diff --git a/.gitignore b/.gitignore index 026703d..d82d9b0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ out/ .env .env.* !.env.example +backend/secrets/ # OS / tooling .DS_Store diff --git a/Makefile b/Makefile index 28e6d6f..5fbdb80 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -.PHONY: help setup up redis down data features train policy insights intel pipeline test lint fmt \ +.PHONY: help setup up redis down data data-calibrated features train policy insights mitigation intel intel-compare adversary pipeline test lint fmt \ migrate seed load api replay replay-live verify platform \ - review retrain shadow models promote console console-build smoke \ - demo demo-reset + review retrain shadow models promote console console-build smoke label-realism e2e-phone \ + demo demo-reset external API_PORT ?= 8010 API_URL ?= http://127.0.0.1:$(API_PORT) @@ -40,6 +40,9 @@ down: ## Stop everything (the demo too) and keep the data data: ## Generate the synthetic dataset cd backend && uv run python -m fraudlens.simulator.generate +data-calibrated: ## Generate the Bangladesh-calibrated dataset into data/full_calibrated + cd backend && uv run python -m fraudlens.simulator.generate --profile calibrated + features: ## Replay the dataset through the feature engine cd backend && uv run python -m fraudlens.features.build @@ -52,10 +55,28 @@ policy: ## Evaluate the decision policy and build the similar-case index insights: ## Threshold sweep, drift and fairness tables for the dashboards cd backend && uv run python -m fraudlens.decision.insights +mitigation: ## Fit and measure young-wallet thresholds (policy v3) against v2 (docs/DECISION_POLICY.md §11) + cd backend && uv run python -m fraudlens.decision.mitigation + intel: ## Train and evaluate the scam-message classifier (docs/FRAUD_TAXONOMY.md) cd backend && uv run python -m fraudlens.intel.train -pipeline: data features train policy insights intel ## Data, features, models and policy end to end +intel-compare: ## Compare message models, MiniLM included with the transformer extra (compare.json) + cd backend && uv run --extra transformer python -m fraudlens.intel.compare + +adversary: ## Adaptive scammers vs frozen, retrained and drift-gated models (after `make pipeline`) + cd backend && uv run python -m fraudlens.models.adversary + +label-realism: ## Retrain on reported-only labels, and with recovery methods; compare on ground truth + cd backend && uv run python -m fraudlens.models.label_realism + +pipeline: data features train policy insights mitigation intel ## Data, features, models and policy end to end + +external: ## Public datasets (PaySim, BAF) and bootstrap intervals; downloads ~560 MB once (docs/MODEL_CARD.md) + cd backend && uv run python -m fraudlens.external.download + cd backend && uv run python -m fraudlens.external.paysim + cd backend && uv run python -m fraudlens.external.baf + cd backend && uv run python -m fraudlens.models.bootstrap migrate: ## Bring the database schema up to date cd backend && uv run alembic upgrade head @@ -69,6 +90,12 @@ load: ## Load the historical period into the database (replaces what is there) api: ## Run the API and its stream worker cd backend && uv run uvicorn fraudlens.api:create_app --factory --host 127.0.0.1 --port $(API_PORT) +worker: ## Run one more stream worker next to the API; start as many as needed (docs/SCALING.md) + cd backend && uv run python -m fraudlens.platform.worker --port $${PORT:-8081} + +loadtest: ## Throughput and latency for 1, 2 and 4 workers; RESETS the database (docs/SCALING.md) + cd backend && uv run python -m fraudlens.platform.loadtest --workers 1 2 4 + replay: ## Replay the test period, except its last day, through the event stream (needs `make api`) cd backend && uv run python -m fraudlens.platform.replay --via stream --to-day 118 @@ -104,6 +131,9 @@ console-build: ## Production build of the console smoke: ## Open every console page in a headless browser as each role (needs the API and the console) cd frontend && pnpm exec playwright install chromium-headless-shell && node scripts/smoke.cjs +e2e-phone: ## The customer phone in Bangla and English, and an appeal round trip (needs the API and the console) + cd frontend && pnpm exec playwright install chromium-headless-shell && node scripts/phone-e2e.cjs + test: ## Run backend tests (the platform tests need `make up`) cd backend && uv run pytest -q @@ -111,6 +141,24 @@ lint: ## Lint and format check cd backend && uv run ruff check src tests && uv run ruff format --check src tests cd frontend && pnpm exec tsc --noEmit && pnpm exec eslint . +.PHONY: tls rotate-jwt partner-key keys +JWT_KEYRING ?= secrets/jwt.json +INGEST_KEYRING ?= secrets/ingest.json + +tls: backend/.env ## The demo behind HTTPS with HSTS on https://localhost:8443 (deploy/Caddyfile) + docker compose --profile demo --profile tls up --build + +rotate-jwt: ## New token-signing key; sessions signed with the old one last until they expire + cd backend && mkdir -p -m 700 secrets && uv run python -m fraudlens.platform.keys rotate-jwt $(JWT_KEYRING) + +partner-key: ## New ingest HMAC key: make partner-key PARTNER=upay [CALLBACK=https://...] + cd backend && mkdir -p -m 700 secrets && uv run python -m fraudlens.platform.keys rotate-partner \ + $(INGEST_KEYRING) --partner $(PARTNER) $(if $(CALLBACK),--callback-url $(CALLBACK)) + +keys: ## List the key ids in both keyrings (never the secrets) + cd backend && for f in $(JWT_KEYRING) $(INGEST_KEYRING); do \ + [ -f $$f ] && uv run python -m fraudlens.platform.keys list $$f; done; true + fmt: ## Auto-format cd backend && uv run ruff check --fix src tests && uv run ruff format src tests diff --git a/README.md b/README.md index 002eb1f..fd30f3b 100644 --- a/README.md +++ b/README.md @@ -426,12 +426,13 @@ To run a challenger in shadow mode, set `FRAUDLENS_SHADOW_MODEL_VERSION` to its Check it ```bash -make test # 227 backend tests; the platform tests need `make up` +make test # 230 backend tests; the platform tests need `make up` make lint # ruff, tsc, eslint make smoke # opens every console page as each role in a headless browser +make e2e-phone # the customer phone in Bangla and English, plus an appeal round trip ``` -`make smoke` needs the demo (or `make api` and `make console`) running and `make setup` done. CI (`.github/workflows/ci.yml`) runs the tests and the lint against real Postgres and Redis, builds the console and builds both images. +`make smoke` and `make e2e-phone` need the demo (or `make api` and `make console`) running and `make setup` done. CI (`.github/workflows/ci.yml`) runs the tests and the lint against real Postgres and Redis, builds the console and builds both images. @@ -462,14 +463,18 @@ Keep two browser windows open: one as `analyst1`, one private window as `supervi | [DATA_ASSUMPTIONS.md](docs/DATA_ASSUMPTIONS.md) | What the synthetic world contains and what it leaves out | | [MODEL_CARD.md](docs/MODEL_CARD.md) | Models, results, ablations, fairness, limits | | [DECISION_POLICY.md](docs/DECISION_POLICY.md) | Tiers, rules, thresholds, explanations, the language model's role | +| [BUSINESS_CASE.md](docs/BUSINESS_CASE.md) | The threshold sweep as a monthly P&L in taka, assumptions and sensitivity | | [PLATFORM.md](docs/PLATFORM.md) | API, workflow, security, stream, measured latency | +| [SCALING.md](docs/SCALING.md) | Several stream workers in one consumer group, measured; Kubernetes manifests in `deploy/` | | [FRAUD_TAXONOMY.md](docs/FRAUD_TAXONOMY.md) | Eight kinds of fraud in the Bangladesh context, what detects each, the scam-message classifier and its limits | | [DEMO_SCRIPT.md](docs/DEMO_SCRIPT.md) | The walk-through | +| [INGEST.md](docs/INGEST.md) | The signed webhook for core banking: ISO 20022 pacs.008 mapping, HMAC signing, replay protection, decision callbacks, a partner SDK | +| [SECURITY.md](docs/SECURITY.md) | STRIDE threat model with the status of each mitigation, personal data and retention | ## ⚠️ What this is not - **Everything runs on synthetic data,** so absolute numbers will not transfer to real traffic. Fraud is more common and more scripted here than in real life, and thresholds must be re-fitted on real data. -- **It is a prototype, not a deployment.** One scorer process holds the feature state, and there is no TLS, secrets manager or horizontal scaling. +- **It is a prototype, not a deployment.** Stream workers scale out, but every one holds the whole feature state and events enter it one at a time, so throughput stops growing at a few workers ([SCALING.md](docs/SCALING.md)). TLS ends at a Caddy proxy, secrets come from files with no vault client, and keys rotate by command, not on a schedule ([SECURITY.md](docs/SECURITY.md)). - **The demo scaffolding is not the product.** The demo accounts, the phone demo endpoints and the review simulator exist only outside production mode. - **No model output moves or blocks money on its own.** A hold waits for an analyst and a freeze needs two people. - **The Bangla texts were written by the developers** and have not been reviewed by a professional translator. diff --git a/backend/alembic/versions/0002_shadow_scores.py b/backend/alembic/versions/0002_shadow_scores.py index 0c57bac..b43db63 100644 --- a/backend/alembic/versions/0002_shadow_scores.py +++ b/backend/alembic/versions/0002_shadow_scores.py @@ -28,7 +28,10 @@ def upgrade() -> None: sa.Column("tier", sa.String(length=8), nullable=False), sa.Column("latency_ms", sa.Float(), nullable=True), sa.Column( - "created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, ), sa.CheckConstraint( "tier IN ('allow', 'warn', 'step_up', 'hold')", name=op.f("ck_shadow_scores_tier") diff --git a/backend/alembic/versions/0004_customer_appeals.py b/backend/alembic/versions/0004_customer_appeals.py new file mode 100644 index 0000000..cbd6bf4 --- /dev/null +++ b/backend/alembic/versions/0004_customer_appeals.py @@ -0,0 +1,78 @@ +"""customer appeals + +Revision ID: 0004_appeals +Revises: 0003 +Create Date: 2026-10-07 12:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "0004_appeals" +down_revision: str | Sequence[str] | None = "0003" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "appeals", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("txn_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.String(length=32), nullable=False), + sa.Column("case_id", sa.Integer(), nullable=True), + sa.Column("tier", sa.String(length=8), nullable=False), + sa.Column("relation", sa.String(length=16), nullable=False), + sa.Column("reason", sa.Text(), nullable=False), + sa.Column("status", sa.String(length=12), server_default="pending", nullable=False), + sa.Column("filed_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("sla_due_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("decided_by", sa.Integer(), nullable=True), + sa.Column("decision_note", sa.Text(), nullable=True), + sa.Column("decided_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "(status = 'pending') = (decided_by IS NULL)", + name=op.f("ck_appeals_decided_has_decider"), + ), + sa.CheckConstraint( + "relation IN ('family', 'friend', 'business', 'seller', 'landlord', 'employer', " + "'other', 'none')", + name=op.f("ck_appeals_relation"), + ), + sa.CheckConstraint( + "status IN ('pending', 'approved', 'rejected')", name=op.f("ck_appeals_status") + ), + sa.CheckConstraint("tier IN ('warn', 'step_up', 'hold')", name=op.f("ck_appeals_tier")), + sa.ForeignKeyConstraint(["case_id"], ["cases.id"], name=op.f("fk_appeals_case_id_cases")), + sa.ForeignKeyConstraint( + ["decided_by"], ["users.id"], name=op.f("fk_appeals_decided_by_users") + ), + sa.ForeignKeyConstraint( + ["txn_id"], + ["transactions.txn_id"], + name=op.f("fk_appeals_txn_id_transactions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_appeals")), + sa.UniqueConstraint("txn_id", name=op.f("uq_appeals_txn_id")), + ) + op.create_index("ix_appeals_status_sla", "appeals", ["status", "sla_due_at"], unique=False) + op.create_index(op.f("ix_appeals_wallet_id"), "appeals", ["wallet_id"], unique=False) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index(op.f("ix_appeals_wallet_id"), table_name="appeals") + op.drop_index("ix_appeals_status_sla", table_name="appeals") + op.drop_table("appeals") diff --git a/backend/alembic/versions/0005_pending_decisions.py b/backend/alembic/versions/0005_pending_decisions.py new file mode 100644 index 0000000..697e232 --- /dev/null +++ b/backend/alembic/versions/0005_pending_decisions.py @@ -0,0 +1,44 @@ +"""pending decisions + +Revision ID: 0005 +Revises: 0004_appeals +Create Date: 2026-10-07 09:30:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0005" +down_revision: str | Sequence[str] | None = "0004_appeals" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "pending_decisions", + sa.Column("txn_id", sa.BigInteger(), nullable=False), + sa.Column("features", postgresql.ARRAY(sa.Float(precision=53)), nullable=False), + sa.Column("context", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False + ), + sa.ForeignKeyConstraint( + ["txn_id"], + ["transactions.txn_id"], + name=op.f("fk_pending_decisions_txn_id_transactions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("txn_id", name=op.f("pk_pending_decisions")), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table("pending_decisions") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 87b2f55..92e59eb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,11 @@ dependencies = [ "uvicorn[standard]>=0.54.0", ] +[project.optional-dependencies] +transformer = [ + "sentence-transformers>=3.0", +] + [build-system] requires = ["uv_build>=0.12.0,<0.13.0"] build-backend = "uv_build" diff --git a/backend/src/fraudlens/api/app.py b/backend/src/fraudlens/api/app.py index d3d3ed6..aefe654 100644 --- a/backend/src/fraudlens/api/app.py +++ b/backend/src/fraudlens/api/app.py @@ -4,6 +4,7 @@ import logging import os +import threading from contextlib import asynccontextmanager import redis.asyncio as aioredis @@ -19,6 +20,7 @@ from ..config import Settings from ..intel.text import load_text_model from ..platform.audit import WorkflowError +from ..platform.callbacks import Dispatcher from ..platform.db import make_engine, make_sessions from ..platform.graph import Graph from ..platform.scoring import Scorer @@ -26,7 +28,21 @@ from ..platform.stream import Worker from .deps import Platform from .middleware import RequestContext, error_body -from .routes import alerts, auth, cases, customer, demo, intel, mlops, network, ops, scoring +from .routes import ( + alerts, + appeals, + auth, + cases, + consortium, + customer, + demo, + ingest, + intel, + mlops, + network, + ops, + scoring, +) log = logging.getLogger(__name__) @@ -79,6 +95,7 @@ def build_platform(settings: Settings) -> Platform: message_limit=RateLimiter(redis, "message-check", 20, 60), # Tighter: each call is a question about the ledger. proof_limit=RateLimiter(redis, "payment-verify", 10, 60), + callbacks=Dispatcher(settings, sessions, redis) if settings.ingest_keyring else None, ) @@ -96,10 +113,24 @@ async def lifespan(app: FastAPI): app.state.aredis = aioredis.from_url(settings.redis_url, decode_responses=True) if settings.run_worker: platform.worker.start() + if platform.callbacks is not None: + platform.callbacks.start() + # Stream workers in other processes change the state too: follow the log. + following = threading.Event() + if settings.follow_interval_s > 0: + threading.Thread( + target=platform.scorer.follow, + args=(following, settings.follow_interval_s), + name="fraudlens-follower", + daemon=True, + ).start() log.info("fraudlens api ready: scoring in %s mode", platform.scorer.mode) try: yield finally: + if platform.callbacks is not None: + platform.callbacks.stop() + following.set() platform.worker.stop() if platform.worker.is_alive(): platform.worker.join(timeout=5.0) @@ -165,7 +196,20 @@ async def unavailable(request: Request, exc: Exception) -> JSONResponse: for kind in (OperationalError, InterfaceError, RedisError): app.add_exception_handler(kind, unavailable) - for module in (auth, scoring, alerts, cases, network, customer, intel, ops, mlops): + for module in ( + auth, + scoring, + ingest, + alerts, + cases, + appeals, + network, + customer, + intel, + ops, + mlops, + consortium, + ): app.include_router(module.router, prefix="/v1") if not settings.production: # stand-ins for the customer app; see routes/demo.py app.include_router(demo.router, prefix="/v1") diff --git a/backend/src/fraudlens/api/deps.py b/backend/src/fraudlens/api/deps.py index 8c8b79d..d7176f1 100644 --- a/backend/src/fraudlens/api/deps.py +++ b/backend/src/fraudlens/api/deps.py @@ -15,10 +15,17 @@ from ..config import Settings from ..intel.text import TextModel from ..platform.audit import Ctx, WorkflowError +from ..platform.callbacks import Dispatcher from ..platform.graph import Graph from ..platform.models import User from ..platform.scoring import Scorer -from ..platform.security import RateLimiter, TokenError, is_revoked, read_token +from ..platform.security import ( + RateLimiter, + TokenError, + client_address, + is_revoked, + read_token, +) from ..platform.stream import Worker REVIEWERS = ("analyst", "supervisor") @@ -47,6 +54,7 @@ class Platform: intel: TextModel | None # the scam-message classifier, when it has been trained message_limit: RateLimiter proof_limit: RateLimiter + callbacks: Dispatcher | None = None # decision callbacks to ingest partners, when configured def platform(request: Request) -> Platform: @@ -67,7 +75,10 @@ def get_session(p: Plat) -> Iterator[Session]: def client_ip(request: Request) -> str | None: - return request.client.host if request.client else None + """The caller's address, through the reverse proxies in FRAUDLENS_TRUSTED_PROXIES.""" + peer = request.client.host if request.client else None + trusted = request.app.state.platform.settings.trusted_proxies + return client_address(peer, request.headers.getlist("x-forwarded-for"), trusted) def current_user( diff --git a/backend/src/fraudlens/api/routes/appeals.py b/backend/src/fraudlens/api/routes/appeals.py new file mode 100644 index 0000000..1c8abf6 --- /dev/null +++ b/backend/src/fraudlens/api/routes/appeals.py @@ -0,0 +1,134 @@ +"""Customer appeals: filed by the customer's app, answered by a reviewer within a deadline.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from fastapi import APIRouter, Query +from sqlalchemy import case as sql_case +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ...platform import appeals as workflow +from ...platform.audit import WorkflowError +from ...platform.models import Appeal, Transaction, User +from ..deps import Db, Plat, Platform, Reviewer, RowId, Service, TxnId +from ..schemas import AppealDecision, AppealIn +from ..views import appeal_view +from .customer import _limit + +router = APIRouter(tags=["appeals"]) + +_TIER = sql_case({"hold": 3, "step_up": 2, "warn": 1}, value=Appeal.tier, else_=0) + + +def customer_appeal_view(p: Platform, s: Session, appeal: Appeal) -> dict: + """What the customer's app shows: where the appeal stands, never who or why internally.""" + txn = s.get(Transaction, appeal.txn_id) + return { + "id": appeal.id, + "txn_id": appeal.txn_id, + "tier": appeal.tier, + "status": appeal.status, + "filed_at": appeal.filed_at, + "sla_due_at": appeal.sla_due_at, + "decided_at": appeal.decided_at, + "txn_status": txn.status if txn else None, + "now": p.scorer.now(), + } + + +def find_appeal(s: Session, wallet_id: str, txn_id: int) -> Appeal: + appeal = s.scalar(select(Appeal).where(Appeal.txn_id == txn_id)) + # The same answer for "no appeal" and "not yours". + if appeal is None or appeal.wallet_id != wallet_id: + raise WorkflowError(404, "appeal_not_found", "no appeal for this payment") + return appeal + + +# ------------------------------------------------------------- the customer's app + + +@router.post("/customer/transactions/{txn_id}/appeal", status_code=201) +def file_appeal(txn_id: TxnId, body: AppealIn, p: Plat, s: Db, ctx: Service) -> dict: + """'This is a genuine payment.' A person answers it; it decides nothing by itself.""" + _limit(p.report_limit, f"appeal:{body.wallet_id}") + appeal = workflow.file_appeal(p.scorer, ctx, body.wallet_id, txn_id, body.relation, body.reason) + return customer_appeal_view(p, s, appeal) + + +@router.get("/customer/transactions/{txn_id}/appeal") +def appeal_status( + txn_id: TxnId, + p: Plat, + s: Db, + ctx: Service, + wallet_id: Annotated[str, Query(pattern=r"^[A-Za-z0-9_-]{1,32}$")], +) -> dict: + return customer_appeal_view(p, s, find_appeal(s, wallet_id, txn_id)) + + +# ------------------------------------------------------------------ the reviewers + + +@router.get("/appeals") +def list_appeals( + p: Plat, + s: Db, + ctx: Reviewer, + status: Literal["pending", "approved", "rejected"] | None = "pending", + limit: Annotated[int, Query(ge=1, le=200)] = 50, +) -> dict: + """The queue: money waiting first, then the nearest deadline.""" + where = [Appeal.status == status] if status else [] + rows = s.execute( + select(Appeal, Transaction) + .join(Transaction, Transaction.txn_id == Appeal.txn_id) + .where(*where) + .order_by( + (Appeal.status != "pending"), + (Transaction.status != "held"), + _TIER.desc(), + Appeal.sla_due_at.asc(), + Appeal.id.desc(), + ) + .limit(limit) + ).all() + ids = {a.decided_by for a, _ in rows if a.decided_by is not None} + names = ( + dict(s.execute(select(User.id, User.username).where(User.id.in_(ids))).all()) if ids else {} + ) + counts = dict(s.execute(select(Appeal.status, func.count()).group_by(Appeal.status)).all()) + now = p.scorer.now() + return { + "now": now, + "counts": {k: counts.get(k, 0) for k in ("pending", "approved", "rejected")}, + "appeals": [ + appeal_view(a, now, names) + | { + "amount": t.amount, + "receiver_id": t.receiver_id, + "txn_status": t.status, + "ts": t.ts, + } + for a, t in rows + ], + } + + +@router.post("/appeals/{appeal_id}/approve") +def approve(appeal_id: RowId, body: AppealDecision, p: Plat, ctx: Reviewer) -> dict: + """The payment is genuine: a held one is released now.""" + done = workflow.decide_appeal(p.scorer, ctx, appeal_id, True, body.note) + return {"appeal": appeal_view(done["appeal"], p.scorer.now()), **_outcome(done)} + + +@router.post("/appeals/{appeal_id}/reject") +def reject(appeal_id: RowId, body: AppealDecision, p: Plat, ctx: Reviewer) -> dict: + """Not convinced: a held payment stays held, and its case decides.""" + done = workflow.decide_appeal(p.scorer, ctx, appeal_id, False, body.note) + return {"appeal": appeal_view(done["appeal"], p.scorer.now()), **_outcome(done)} + + +def _outcome(done: dict) -> dict: + return {"txn_status": done["txn_status"], "released": done["released"]} diff --git a/backend/src/fraudlens/api/routes/consortium.py b/backend/src/fraudlens/api/routes/consortium.py new file mode 100644 index 0000000..8c4ab35 --- /dev/null +++ b/backend/src/fraudlens/api/routes/consortium.py @@ -0,0 +1,120 @@ +"""The cross-provider mule-intelligence consortium (simulated; see docs/CONSORTIUM.md). + +Read-only views of partner feeds, matches and the hub's audit chain, a receiver +lookup, and the dispute path. The state comes from `artifacts/consortium/`, written +by `python -m fraudlens.consortium.simulate`; without it these routes answer 404 and +nothing else in the platform changes. +""" + +from __future__ import annotations + +import threading +from typing import Annotated, Literal + +from fastapi import APIRouter, Path, Query +from pydantic import BaseModel, Field + +from ...consortium.demo import ConsortiumService +from ...consortium.protocol import ProtocolError +from ...platform.audit import WorkflowError, audit +from ...platform.events import Identifier +from ...platform.scoring import clean +from ..deps import Db, Plat, Reviewer, Supervisor + +router = APIRouter(prefix="/consortium", tags=["consortium"]) + +_lock = threading.Lock() +_services: dict[str, ConsortiumService | None] = {} + + +def service(p: Plat) -> ConsortiumService: + key = str(p.settings.artifacts_dir) + with _lock: + if key not in _services: + _services[key] = ConsortiumService.load(p.settings.artifacts_dir) + svc = _services[key] + if svc is None: + raise WorkflowError(404, "consortium_not_built", + "run `python -m fraudlens.consortium.simulate` first") # fmt: skip + return svc + + +class Lookup(BaseModel): + wallet_id: Identifier + + +class DisputeIn(BaseModel): + listing_id: str = Field(pattern=r"^[a-z]{2,16}:\d{1,8}$") + raised_by: str = Field(pattern=r"^[a-z]{2,16}$") + reason: str = Field(min_length=3, max_length=500) + + +class Resolve(BaseModel): + outcome: Literal["upheld", "withdrawn"] + + +def _protocol(exc: ProtocolError) -> WorkflowError: + return WorkflowError(409, "consortium_refused", str(exc)) + + +@router.get("") +def overview(p: Plat, ctx: Reviewer) -> dict: + """Members, their latest signed feeds, the privacy guarantees and the measured results.""" + return clean(service(p).overview()) + + +@router.get("/matches") +def matches(p: Plat, ctx: Reviewer, + limit: Annotated[int, Query(ge=1, le=500)] = 200) -> list[dict]: # fmt: skip + """Test-period receivers that matched a partner listing (one row per wallet).""" + return clean(service(p).matches(limit)) + + +@router.get("/audit") +def hub_audit(p: Plat, ctx: Reviewer, + limit: Annotated[int, Query(ge=1, le=500)] = 100) -> list[dict]: # fmt: skip + """The hub's hash-chained log, newest first: counts and ids, never identifiers.""" + return clean(service(p).audit(limit)) + + +@router.post("/lookup") +def lookup(body: Lookup, p: Plat, s: Db, ctx: Reviewer) -> dict: + svc = service(p) + try: + out = svc.lookup(body.wallet_id) + except ProtocolError as exc: + raise WorkflowError(404, "not_in_consortium", str(exc)) from exc + audit(s, ctx, "consortium.lookup", "wallet", body.wallet_id, + matches=len(out["matches"]), signal=out["signal"]) # fmt: skip + s.commit() + return clean(out) + + +@router.post("/disputes", status_code=201) +def open_dispute(body: DisputeIn, p: Plat, s: Db, ctx: Reviewer) -> dict: + """Contest a listing: it stops counting for every member at once.""" + try: + out = service(p).open_dispute(body.raised_by, body.listing_id, body.reason) + except ProtocolError as exc: + raise _protocol(exc) from exc + audit(s, ctx, "consortium.dispute", "listing", body.listing_id, dispute=out["dispute_id"]) + s.commit() + return clean(out) + + +@router.post("/disputes/{dispute_id}/resolve") +def resolve( + dispute_id: Annotated[str, Path(pattern=r"^D\d{4}$")], + body: Resolve, + p: Plat, + s: Db, + ctx: Supervisor, +) -> dict: + """Decide a dispute for the listing member: keep the listing, or withdraw it for good.""" + try: + out = service(p).resolve_dispute(dispute_id, body.outcome) + except ProtocolError as exc: + raise _protocol(exc) from exc + audit(s, ctx, "consortium.resolve", "dispute", dispute_id, outcome=body.outcome) + s.commit() + return clean(out) diff --git a/backend/src/fraudlens/api/routes/customer.py b/backend/src/fraudlens/api/routes/customer.py index c5eb4e7..ad84859 100644 --- a/backend/src/fraudlens/api/routes/customer.py +++ b/backend/src/fraudlens/api/routes/customer.py @@ -11,6 +11,7 @@ from ...intel import proof from ...intel.analyze import check_message +from ...intel.memory import explain as explain_link from ...intel.taxonomy import load_taxonomy from ...platform import cases as workflow from ...platform.audit import WorkflowError @@ -37,10 +38,10 @@ def recipient_check(body: RecipientCheck, p: Plat, ctx: Service) -> dict: it cannot be used to probe which wallets have been caught. """ _limit(p.check_limit, body.sender_id) - return check_recipient(p, body.receiver_id) + return check_recipient(p, body.receiver_id, body.sender_id) -def check_recipient(p: Platform, receiver_id: str) -> dict: +def check_recipient(p: Platform, receiver_id: str, sender_id: str | None = None) -> dict: risk = p.graph.risk(receiver_id) level = "none" if risk is not None: @@ -48,8 +49,16 @@ def check_recipient(p: Platform, receiver_id: str) -> dict: level = "high" elif risk["mule_alert"]: level = "caution" + # The sender's own flagged message named this wallet: say so before they type an amount. + # `reasons` appears only then, so an ordinary answer is unchanged and reveals nothing new. + link = None + if sender_id is not None: + link = p.scorer.messages.link(sender_id, receiver_id, None, p.scorer.now()) + if link is not None and level == "none": + level = "caution" message = p.scorer.policy.messages["scam"]["warn"].model_dump() if level != "none" else None - return {"receiver_id": receiver_id, "level": level, "message": message} + out = {"receiver_id": receiver_id, "level": level, "message": message} + return out if link is None else out | {"reasons": [explain_link(link)]} @router.post("/transactions/{txn_id}/respond") @@ -80,11 +89,16 @@ def message_check(body: MessageCheck, p: Plat, ctx: Service) -> dict: the text is not kept. What the customer is told is the taxonomy's fixed advice. """ _limit(p.message_limit, body.wallet_id) - return check_text(p, body.text) + return check_text(p, body.text, body.wallet_id) -def check_text(p: Platform, text: str) -> dict: - return check_message(text, p.intel, load_taxonomy()) +def check_text(p: Platform, text: str, wallet_id: str | None = None) -> dict: + """With `wallet_id`, a flagged message's wallet IDs and amounts (never its text) are + kept for half an hour, so a payment that follows it is warned (`intel/memory.py`).""" + found = check_message(text, p.intel, load_taxonomy()) + if wallet_id is not None: + p.scorer.messages.remember(wallet_id, text, found, p.scorer.now()) + return found @router.post("/payment-verify") diff --git a/backend/src/fraudlens/api/routes/demo.py b/backend/src/fraudlens/api/routes/demo.py index f47515e..bf1abde 100644 --- a/backend/src/fraudlens/api/routes/demo.py +++ b/backend/src/fraudlens/api/routes/demo.py @@ -21,12 +21,14 @@ from ...decision.policy import TIERS from ...features.engine import DISTRICT_COORDS, MIN_NETWORK_HISTORY +from ...platform import appeals as appeal_flow from ...platform import cases as workflow from ...platform.audit import WorkflowError, audit from ...platform.events import Identifier, TxnIn from ...platform.models import Decision, Transaction from ..deps import Db, Plat, Platform, Staff from ..schemas import ( + DemoAppeal, DemoClock, DemoPayment, DemoResponse, @@ -36,6 +38,7 @@ ScamReport, ) from ..views import result_view +from .appeals import customer_appeal_view, find_appeal from .customer import _limit, check_recipient, check_text, verify_payment router = APIRouter(prefix="/demo", tags=["demo"]) @@ -212,17 +215,41 @@ def report(body: ScamReport, p: Plat, ctx: Staff) -> dict: return {"report_id": made.id, "case_id": made.case_id} +def _sender(s: Session, txn_id: int) -> str: + sender = s.scalar(select(Transaction.sender_id).where(Transaction.txn_id == txn_id)) + if sender is None: + raise WorkflowError(404, "txn_not_found", "no such transaction") + return sender + + +@router.post("/appeal", status_code=201) +def appeal(body: DemoAppeal, p: Plat, s: Db, ctx: Staff) -> dict: + """The demo customer's 'this is a genuine payment'. A reviewer answers it on /appeals.""" + sender = _sender(s, body.txn_id) + _limit(p.report_limit, f"appeal:{sender}") + made = appeal_flow.file_appeal(p.scorer, ctx, sender, body.txn_id, body.relation, body.reason) + return customer_appeal_view(p, s, made) + + +@router.get("/appeal") +def appeal_status( + txn_id: Annotated[int, Query(ge=0, lt=2**62)], p: Plat, s: Db, ctx: Staff +) -> dict: + """Where the demo customer's appeal stands, as their app would show it.""" + return customer_appeal_view(p, s, find_appeal(s, _sender(s, txn_id), txn_id)) + + @router.post("/recipient-check") def recipient_check(body: RecipientCheck, p: Plat, ctx: Staff) -> dict: """What the app would show before the customer types an amount.""" - return check_recipient(p, body.receiver_id) + return check_recipient(p, body.receiver_id, body.sender_id) @router.post("/message-check") def message_check(body: MessageCheck, p: Plat, s: Db, ctx: Staff) -> dict: """The demo customer asks whether a message is a scam. The text is not recorded.""" _limit(p.message_limit, body.wallet_id) - found = check_text(p, body.text) + found = check_text(p, body.text, body.wallet_id) audit( s, ctx, "demo.message_checked", "wallet", body.wallet_id, level=found["level"], categories=[c["id"] for c in found["categories"]], diff --git a/backend/src/fraudlens/api/routes/ingest.py b/backend/src/fraudlens/api/routes/ingest.py new file mode 100644 index 0000000..f5ac6bb --- /dev/null +++ b/backend/src/fraudlens/api/routes/ingest.py @@ -0,0 +1,181 @@ +"""The signed webhook a core-banking system pushes transactions to (docs/INGEST.md). + +No bearer token: the partner signs each request with its HMAC key. The raw body is +read before anything is parsed, because the signature is over the bytes sent. +""" + +from __future__ import annotations + +import time +from typing import Literal + +from fastapi import APIRouter, Query, Request +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import ValidationError +from starlette.concurrency import run_in_threadpool + +from ...platform import callbacks +from ...platform.audit import WorkflowError +from ...platform.ingest import ( + IDEMPOTENCY_HEADER, + KEY_HEADER, + NONCE_HEADER, + SIGNATURE_HEADER, + TIMESTAMP_HEADER, + Idempotency, + IngestInvalid, + Pacs008, + claim_nonce, + first_sightings, + forget_sightings, + parse, + verify, +) +from ...platform.keys import Key, load_keyring +from ...platform.stream import MAX_BACKLOG, enqueue +from ..deps import Platform +from ..views import result_view + +router = APIRouter(prefix="/ingest", tags=["ingest"]) +_ERROR = "__error__" # a stored refusal, replayed as the same error + +_SIGNED = { + "parameters": [ + { + "name": h, + "in": "header", + "required": h != IDEMPOTENCY_HEADER, + "schema": {"type": "string"}, + } + for h in (KEY_HEADER, TIMESTAMP_HEADER, NONCE_HEADER, SIGNATURE_HEADER, IDEMPOTENCY_HEADER) + ], + "requestBody": { + "required": True, + "content": {"application/json": {"schema": Pacs008.model_json_schema()}}, + }, +} + + +def _authenticate(p: Platform, request: Request, body: bytes) -> Key: + if p.settings.ingest_keyring is None: + raise WorkflowError(404, "not_found", "ingest is not configured on this service") + path = request.url.path + (f"?{request.url.query}" if request.url.query else "") + window = p.settings.ingest_window_seconds + key = verify( + load_keyring(p.settings.ingest_keyring), request.headers, request.method, path, body, window + ) + claim_nonce(p.redis, key, request.headers[NONCE_HEADER], window) + return key + + +def _parse(body: bytes, single: bool): + try: + return parse(body, single) + except ValidationError as exc: + raise RequestValidationError(exc.errors()) from None + except IngestInvalid as exc: + raise RequestValidationError(exc.errors) from None + + +def _queue(p: Platform, key: Key, message: Pacs008, txns: list) -> dict: + backlog = p.worker.backlog() + if backlog > MAX_BACKLOG: + raise WorkflowError( + 503, "backlog_full", "the event queue is full; retry later", + retry_after_seconds=30, backlog=backlog, + ) # fmt: skip + ids = [t.txn_id for t in txns] + new = first_sightings(p.redis, ids) + fresh = [t for t, n in zip(txns, new, strict=True) if n] + try: + enqueue(p.redis, p.settings.events_stream, fresh) + except Exception: + forget_sightings(p.redis, [t.txn_id for t in fresh]) + raise + ring = load_keyring(p.settings.ingest_keyring) + if ring.callback_url(key.partner): + now = time.time() + for transfer, is_new in zip(message.CdtTrfTxInf, new, strict=True): + if is_new: + callbacks.schedule( + p.redis, key.partner, int(transfer.PmtId.TxId), transfer.PmtId.EndToEndId, now + ) + return { + "msg_id": message.GrpHdr.MsgId, + "accepted": len(fresh), + "duplicates": [ + t.PmtId.EndToEndId for t, n in zip(message.CdtTrfTxInf, new, strict=True) if not n + ], + "callback": bool(ring.callback_url(key.partner)), + } + + +def _decide(p: Platform, message: Pacs008, txns: list) -> dict: + [new] = first_sightings(p.redis, [txns[0].txn_id]) # a later queued copy is a duplicate + try: + [result] = p.scorer.process([txns[0].event()]) + except Exception: + if new: + forget_sightings(p.redis, [txns[0].txn_id]) + raise + if result.status == "stale": + raise WorkflowError( + 409, "stale_event", "this transaction is too far behind the stream to be scored" + ) + return {"end_to_end_id": message.CdtTrfTxInf[0].PmtId.EndToEndId, **result_view(result)} + + +async def _handle(request: Request, single: bool, wait: bool) -> JSONResponse: + p: Platform = request.app.state.platform + body = await request.body() + + def work() -> tuple[int, dict, bool]: + key = _authenticate(p, request, body) + idem = Idempotency(p.redis, key.partner, request.headers.get(IDEMPOTENCY_HEADER), body) + replay = idem.begin() + if replay is not None: + return *replay, True + try: + message, txns = _parse(body, single) + if wait: + status, answer = 200, jsonable_encoder(_decide(p, message, txns)) + else: + status, answer = 202, _queue(p, key, message, txns) + except WorkflowError as exc: + if exc.status >= 500: + idem.abandon() + else: # a refusal is an answer too: the same request gets it again + idem.finish(exc.status, {_ERROR: [exc.status, exc.code, exc.message]}) + raise + except Exception: # invalid, or a backing service failed: a retry may run it again + idem.abandon() + raise + idem.finish(status, answer) + return status, answer, False + + status, answer, replayed = await run_in_threadpool(work) + if _ERROR in answer: + code_status, code, message = answer[_ERROR] + raise WorkflowError(code_status, code, message) + headers = {"Idempotent-Replayed": "true"} if replayed else {} + return JSONResponse(answer, status, headers=headers) + + +@router.post("/transactions", status_code=202, openapi_extra=_SIGNED) +async def ingest_one( + request: Request, + wait: Literal["decision"] | None = Query( + default=None, description="`decision`: score now and answer with it (200)" + ), +) -> JSONResponse: + """One pacs.008 credit transfer, HMAC-signed by a partner. Queued for the stream + (202) or, with `?wait=decision`, decided now (200, the same answer as /v1/score).""" + return await _handle(request, single=True, wait=wait == "decision") + + +@router.post("/transactions/batch", status_code=202, openapi_extra=_SIGNED) +async def ingest_batch(request: Request) -> JSONResponse: + """Up to 500 credit transfers in one signed pacs.008 message, queued in order.""" + return await _handle(request, single=False, wait=False) diff --git a/backend/src/fraudlens/api/routes/ops.py b/backend/src/fraudlens/api/routes/ops.py index 9bc3ab1..6a4b501 100644 --- a/backend/src/fraudlens/api/routes/ops.py +++ b/backend/src/fraudlens/api/routes/ops.py @@ -11,9 +11,12 @@ from fastapi import APIRouter, Query, Response from sqlalchemy import Date, cast, func, select, text +from ...decision.business import Assumptions, business_case from ...decision.evaluate import REPORT_FILE as POLICY_REPORT_FILE from ...decision.insights import INSIGHTS_FILE +from ...decision.mitigation import MITIGATION_FILE from ...models import registry +from ...models.label_realism import REPORT_FILE as LABEL_REALISM_FILE from ...platform.audit import WorkflowError from ...platform.models import AuditLog, Case, Decision, FreezeRequest, Transaction from ..deps import Db, Oversight, Plat, Staff @@ -249,12 +252,19 @@ def model_report( """How a model version (default: the served one) and the policy did on the test period. A back-test with known labels: `model` is the model card's numbers, `policy` the - effect of the rules, `insights` the threshold sweep, drift and fairness tables. + effect of the rules, `insights` the threshold sweep, drift and fairness tables, + `mitigation` the young-wallet segment thresholds measured before and after. """ if version is None and p.scorer.bundle is not None: version = p.scorer.bundle.version if version is None: - return {"model_version": None, "model": None, "policy": None, "insights": None} + return { + "model_version": None, + "model": None, + "policy": None, + "insights": None, + "mitigation": None, + } if version not in registry.versions(p.settings.models_dir): raise WorkflowError(404, "model_not_found", f"no model version {version}") directory = p.settings.models_dir / version @@ -263,9 +273,53 @@ def model_report( "model": _report(directory, "report.json"), "policy": _report(directory, POLICY_REPORT_FILE), "insights": _report(directory, INSIGHTS_FILE), + "label_realism": _label_realism(p.settings.artifacts_dir), + "mitigation": _report(directory, MITIGATION_FILE), } +def _business(p: Plat, version: str | None, assumptions: Assumptions) -> dict: + if version is None and p.scorer.bundle is not None: + version = p.scorer.bundle.version + if version is None or version not in registry.versions(p.settings.models_dir): + raise WorkflowError(404, "model_not_found", f"no model version {version}") + insights = _report(p.settings.models_dir / version, INSIGHTS_FILE) + if not insights or not insights.get("impact"): + raise WorkflowError(404, "not_evaluated", f"model {version} has no threshold sweep") + return business_case(insights, assumptions) + + +@router.get("/model/business") +def business( + p: Plat, ctx: Staff, version: Annotated[str | None, Query(pattern=r"^v\d{1,6}$")] = None +) -> dict: + """The threshold sweep as a monthly profit and loss in taka, under default assumptions. + + Every assumption, its source and its range are in the answer; POST to change them. + """ + return _business(p, version, Assumptions()) + + +@router.post("/model/business") +def business_with( + assumptions: Assumptions, + p: Plat, + ctx: Staff, + version: Annotated[str | None, Query(pattern=r"^v\d{1,6}$")] = None, +) -> dict: + """The same, with some assumptions changed. Computes only: nothing is stored.""" + return _business(p, version, assumptions) + + +def _label_realism(artifacts: Path) -> dict | None: + """The under-reporting experiment (`make label-realism`), averaged over seeds.""" + report = _report(artifacts / "reports", LABEL_REALISM_FILE) + if report is None: + return None + keep = ("created_at", "seeds", "operating_point", "regimes", "summary") + return {key: report.get(key) for key in keep} + + @router.get("/policy") def policy(p: Plat, ctx: Staff) -> dict: """The whole policy as served: tiers, rules with their conditions, texts, fallback.""" diff --git a/backend/src/fraudlens/api/schemas.py b/backend/src/fraudlens/api/schemas.py index fd792de..707b90d 100644 --- a/backend/src/fraudlens/api/schemas.py +++ b/backend/src/fraudlens/api/schemas.py @@ -121,6 +121,32 @@ class ScamReport(Body): ] +AppealReason = Annotated[ + str, StringConstraints(strip_whitespace=True, min_length=3, max_length=500) +] +Relation = Literal[ + "family", "friend", "business", "seller", "landlord", "employer", "other", "none" +] + + +class AppealIn(Body): + """'This is a genuine payment.' Typed by the customer: shown masked, never read to decide.""" + + wallet_id: Identifier + relation: Relation + reason: AppealReason + + +class DemoAppeal(Body): + txn_id: int = Field(ge=0, lt=2**62) + relation: Relation + reason: AppealReason + + +class AppealDecision(Body): + note: Reason + + class MessageCheck(Body): """A message the customer received and wants checked. It is read, never stored.""" diff --git a/backend/src/fraudlens/api/views.py b/backend/src/fraudlens/api/views.py index beb0594..c2be440 100644 --- a/backend/src/fraudlens/api/views.py +++ b/backend/src/fraudlens/api/views.py @@ -6,7 +6,8 @@ from ..intel.attribute import categorise from ..intel.taxonomy import load_taxonomy -from ..platform.models import Case, CaseEvent, Decision, FreezeRequest, Transaction, User +from ..platform.models import Appeal, Case, CaseEvent, Decision, FreezeRequest, Transaction, User +from ..platform.pii import redact from ..platform.scoring import Result @@ -74,6 +75,28 @@ def freeze_view(r: FreezeRequest, names: dict[int, str] | None = None) -> dict: } +def appeal_view(a: Appeal, now: datetime, names: dict[int, str] | None = None) -> dict: + names = names or {} + return { + "id": a.id, + "txn_id": a.txn_id, + "wallet_id": a.wallet_id, + "case_id": a.case_id, + "tier": a.tier, + "relation": a.relation, + # Typed by a customer: numbers and addresses in it are masked. + "reason": redact(a.reason), + "status": a.status, + "filed_at": a.filed_at, + "sla_due_at": a.sla_due_at, + "overdue": a.status == "pending" and a.sla_due_at < now, + "decided_by": a.decided_by, + "decider": names.get(a.decided_by), + "decision_note": a.decision_note, + "decided_at": a.decided_at, + } + + def alert_view(t: Transaction, d: Decision) -> dict: """One line of the alert queue.""" return { @@ -146,6 +169,7 @@ def decision_view(t: Transaction, d: Decision) -> dict: "recommended_actions", "similar_cases", "headline", + "segment", ) }, # fmt: skip } diff --git a/backend/src/fraudlens/config.py b/backend/src/fraudlens/config.py index 3a3b72d..3be9cc7 100644 --- a/backend/src/fraudlens/config.py +++ b/backend/src/fraudlens/config.py @@ -3,7 +3,9 @@ from pathlib import Path from pydantic import SecretStr -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict + +from .secret_sources import SecretFilesSource BACKEND_DIR = Path(__file__).resolve().parents[2] @@ -16,6 +18,9 @@ class Settings(BaseSettings): data_dir: Path = BACKEND_DIR / "data" artifacts_dir: Path = BACKEND_DIR / "artifacts" dataset: str = "full" # sub-directory of data_dir the platform serves + # Simulator profile for `fraudlens.simulator.generate`: "default" or "calibrated" + # (Bangladesh-sourced parameters, docs/DATA_ASSUMPTIONS.md §11). + sim_profile: str = "default" # Model and policy the API serves. None means the promoted (CURRENT) model version. models_root: Path | None = None @@ -30,6 +35,11 @@ class Settings(BaseSettings): events_stream: str = "fraudlens:events" alerts_channel: str = "fraudlens:alerts" run_worker: bool = True # consume the event stream inside the API process + # Stream workers (`python -m fraudlens.platform.worker`, docs/SCALING.md). + worker_name: str | None = None # consumer name; default host-pid + worker_port: int = 8081 # /health, /ready, /metrics + claim_idle_ms: int = 30_000 # a pending entry idle this long is taken from its consumer + follow_interval_s: float = 0.5 # how often the API's replica catches up with the log # Signing key for access tokens. The default is for local development only; # the API refuses to start with it when environment == "production". @@ -52,6 +62,28 @@ class Settings(BaseSettings): # Where the console is served from in development (`make console`). cors_origins: list[str] = ["http://localhost:3100", "http://127.0.0.1:3100"] + # Signing keys with ids, re-read when the file changes (`fraudlens.platform.keys`). + # When set they replace jwt_secret: the current key signs, unretired ones verify. + jwt_keyring: Path | None = None + # Partner HMAC keys for /v1/ingest (docs/INGEST.md); without them it answers 404. + ingest_keyring: Path | None = None + ingest_window_seconds: int = 300 # how far a request's timestamp may be from ours + # Reverse proxies (CIDRs) whose X-Forwarded-For is believed; nobody else's is. + trusted_proxies: list[str] = [] + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + # FRAUDLENS__FILE and FRAUDLENS_SECRETS_PROVIDER: see secret_sources.py. + files = SecretFilesSource(settings_cls) + return init_settings, env_settings, files, dotenv_settings, file_secret_settings + @property def dataset_dir(self) -> Path: return self.data_dir / self.dataset diff --git a/backend/src/fraudlens/consortium/__init__.py b/backend/src/fraudlens/consortium/__init__.py new file mode 100644 index 0000000..f4a1e96 --- /dev/null +++ b/backend/src/fraudlens/consortium/__init__.py @@ -0,0 +1 @@ +"""Cross-provider mule intelligence that shares no customer data (docs/CONSORTIUM.md).""" diff --git a/backend/src/fraudlens/consortium/crypto.py b/backend/src/fraudlens/consortium/crypto.py new file mode 100644 index 0000000..efaff85 --- /dev/null +++ b/backend/src/fraudlens/consortium/crypto.py @@ -0,0 +1,233 @@ +"""The consortium's cryptography, from the standard library only. + +Three primitives: + +- an **oblivious PRF** (OPRF). A provider turns a wallet number into a token + without the hub seeing the number, and without being able to make tokens on its + own. The hub holds the key `k`; the provider sends the hashed identifier blinded + by a random factor, the hub raises it to `k`, the provider removes the blind. + Because tokens can only be made one rate-limited, audited request at a time, + no member can hash every 01XXXXXXXXX number offline and reverse the tokens. +- **Schnorr signatures**, so a bundle is attributable to the provider that issued + it (the hub relays bundles but cannot forge or alter a listing). +- a **Bloom filter**, for suspected-but-unconfirmed wallets: partners can test a + token against it but cannot list its contents, and a hit is deniable by design. + +Group: the 2048-bit MODP group of RFC 3526 (group 14), a safe prime p = 2q + 1, +working in the subgroup of quadratic residues of prime order q. Secret exponents +are 256 bits (short exponents give 128-bit security in this group). +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import math +import secrets +from dataclasses import dataclass + + +def _pi_bits(bits: int) -> int: + """floor(pi * 2**bits), by Machin's formula on integers.""" + guard = 64 + one = 1 << (bits + guard) + + def arctan_inv(x: int) -> int: + total, term, n, sign = 0, one // x, 1, 1 + while term: + total += sign * (term // n) + term //= x * x + n += 2 + sign = -sign + return total + + return (4 * (4 * arctan_inv(5) - arctan_inv(239))) >> guard + + +# RFC 3526 section 3: p = 2^2048 - 2^1984 - 1 + 2^64 * ( [2^1918 pi] + 124476 ). +P = 2**2048 - 2**1984 - 1 + 2**64 * (_pi_bits(1918) + 124476) +Q = (P - 1) // 2 +G = 4 # 2 squared: a generator of the order-q subgroup +P_BYTES = 256 +EXP_BITS = 256 + +# A cheap check that the constant came out right (the published prime's last bytes). +assert P % 2**64 == 2**64 - 1 and hex(P).startswith("0xffffffffffffffffc90fdaa22168c234") + + +def _expand(domain: bytes, data: bytes, size: int) -> int: + return int.from_bytes(hashlib.shake_256(domain + b"\x00" + data).digest(size), "big") + + +def hash_to_group(data: bytes) -> int: + """Map bytes to the order-q subgroup with no known discrete log.""" + h = _expand(b"FL-CONS-H2G-v1", data, P_BYTES + 16) % P + return pow(h, 2, P) or G # squaring lands in the quadratic residues + + +def _jacobi(a: int, n: int) -> int: + """The Jacobi symbol (a/n) for odd n > 0, by quadratic reciprocity.""" + a %= n + result = 1 + while a: + twos = (a & -a).bit_length() - 1 + a >>= twos + if twos & 1 and n % 8 in (3, 5): + result = -result + if a % 4 == 3 and n % 4 == 3: + result = -result + a, n = n % a, a + return result if n == 1 else 0 + + +def in_group(x: int) -> bool: + """Is x in the order-q subgroup? For a safe prime that means a quadratic residue. + + The same answer as pow(x, Q, P) == 1, far cheaper. + """ + return 1 < x < P and _jacobi(x, P) == 1 + + +def random_exponent() -> int: + return secrets.randbits(EXP_BITS) | 1 << (EXP_BITS - 1) + + +# --------------------------------------------------------------------- OPRF + + +@dataclass(frozen=True) +class OprfKey: + """The hub's secret for one key epoch. `public` lets clients unblind.""" + + epoch: str + secret: int + + @property + def public(self) -> int: + return pow(G, self.secret, P) + + @staticmethod + def generate(epoch: str) -> OprfKey: + return OprfKey(epoch, random_exponent()) + + def evaluate(self, blinded: int) -> int: + """The hub's half: raise a blinded element to the key.""" + if not in_group(blinded): + raise ValueError("not an element of the group") + return pow(blinded, self.secret, P) + + +def blind(element: int) -> tuple[int, int]: + """Client: hide `element` as element * g^r. Returns (blinded, r).""" + r = random_exponent() + return element * pow(G, r, P) % P, r + + +def unblind(evaluated: int, r: int, public: int) -> int: + """Client: (element * g^r)^k / (g^k)^r = element^k.""" + return evaluated * pow(pow(public, r, P), -1, P) % P + + +def token_from_element(element_k: int, epoch: str, kind: str) -> str: + """The shared token: a hash of element^k, bound to the key epoch and identifier kind.""" + raw = element_k.to_bytes(P_BYTES, "big") + digest = hashlib.sha256(b"FL-CONS-TOKEN-v1|" + f"{epoch}|{kind}|".encode() + raw) + return digest.hexdigest()[:32] # 128 bits + + +# --------------------------------------------------------------- signatures + + +@dataclass(frozen=True) +class SigningKey: + secret: int + + @staticmethod + def generate() -> SigningKey: + return SigningKey(random_exponent()) + + @property + def public(self) -> int: + return pow(G, self.secret, P) + + def sign(self, message: bytes) -> str: + # Deterministic nonce (as in RFC 6979): no randomness to get wrong. + nonce_seed = self.secret.to_bytes(P_BYTES, "big") + message + r = _expand(b"FL-CONS-NONCE-v1", nonce_seed, 48) % Q or 1 + commitment = pow(G, r, P) + e = _challenge(commitment, self.public, message) + s = (r + e * self.secret) % Q + return f"{e:064x}.{s:x}" + + +def _challenge(commitment: int, public: int, message: bytes) -> int: + data = commitment.to_bytes(P_BYTES, "big") + public.to_bytes(P_BYTES, "big") + message + return int.from_bytes(hashlib.sha256(b"FL-CONS-SIG-v1" + data).digest(), "big") + + +def verify(public: int, message: bytes, signature: str) -> bool: + try: + e_hex, s_hex = signature.split(".") + e, s = int(e_hex, 16), int(s_hex, 16) + except ValueError: + return False + if not (0 < s < Q and 1 < public < P): + return False + commitment = pow(G, s, P) * pow(pow(public, e, P), -1, P) % P + return hmac.compare_digest(f"{_challenge(commitment, public, message):064x}", e_hex) + + +def fingerprint(public: int) -> str: + """Short, stable name for a public key, for audit lines and screens.""" + return hashlib.sha256(public.to_bytes(P_BYTES, "big")).hexdigest()[:16] + + +# ------------------------------------------------------------- Bloom filter + + +class BloomFilter: + """A set of tokens that can be tested but not listed.""" + + def __init__(self, m: int, k: int, bits: bytearray | None = None, n: int = 0) -> None: + self.m, self.k, self.n = m, k, n + self.bits = bits if bits is not None else bytearray((m + 7) // 8) + + @staticmethod + def sized(capacity: int, fp_rate: float) -> BloomFilter: + capacity = max(capacity, 1) + m = max(1024, math.ceil(-capacity * math.log(fp_rate) / math.log(2) ** 2)) + k = max(1, min(32, round(m / capacity * math.log(2)))) + return BloomFilter(m, k) + + def _positions(self, token: str) -> list[int]: + # k independent positions from one extendable-output hash (double hashing + # modulo a small, even m correlates the positions and inflates false hits). + d = hashlib.shake_256(b"FL-CONS-BLOOM-v1" + token.encode()).digest(8 * self.k) + return [int.from_bytes(d[8 * i : 8 * i + 8], "big") % self.m for i in range(self.k)] + + def add(self, token: str) -> None: + for pos in self._positions(token): + self.bits[pos >> 3] |= 1 << (pos & 7) + self.n += 1 + + def __contains__(self, token: str) -> bool: + return all(self.bits[pos >> 3] >> (pos & 7) & 1 for pos in self._positions(token)) + + def expected_fp_rate(self) -> float: + return (1 - math.exp(-self.k * self.n / self.m)) ** self.k + + def to_dict(self) -> dict: + return { + "m": self.m, + "k": self.k, + "n": self.n, + "bits": base64.b64encode(bytes(self.bits)).decode(), + } + + @staticmethod + def from_dict(d: dict) -> BloomFilter: + bits = bytearray(base64.b64decode(d["bits"])) + if len(bits) != (d["m"] + 7) // 8: + raise ValueError("Bloom filter size does not match its header") + return BloomFilter(int(d["m"]), int(d["k"]), bits, int(d["n"])) diff --git a/backend/src/fraudlens/consortium/demo.py b/backend/src/fraudlens/consortium/demo.py new file mode 100644 index 0000000..c9d481f --- /dev/null +++ b/backend/src/fraudlens/consortium/demo.py @@ -0,0 +1,237 @@ +"""The simulated consortium as the console sees it. + +`save_demo` writes the end state of the headline simulation run to +`artifacts/consortium/`; `ConsortiumService` loads it in the API process, rebuilds +the hub and the four members from it (re-verifying every bundle signature) and +answers lookups and disputes against it. Disputes and lookups made through the API +live in memory and are lost on restart; every one also goes to the platform's audit +log. + +The keys in `demo_keys.json` belong to simulated parties. A real hub keeps its OPRF +key in an HSM and each member keeps its own signing key; nobody holds all of them. +""" + +from __future__ import annotations + +import json +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pandas as pd + +from . import crypto +from .hub import Hub, Member +from .protocol import Bundle, ProtocolError, iso, parse_iso + +GUARANTEES = ( + "No wallet number, handset id, name or NID leaves a provider: bundles carry only " + "128-bit tokens.", + "Tokens come from an oblivious PRF: the hub never sees the number it is tokenising, " + "and members cannot make tokens offline, so they cannot reverse partners' lists by " + "brute force over the 01XXXXXXXXX numbering plan.", + "Token requests are rate-limited per member and audited, which bounds enumeration.", + "Suspected (unconfirmed) wallets are shared only inside a Bloom filter: testable, " + "not listable, and a hit is deniable.", + "Every bundle is signed by the provider that issued it: the hub relays but cannot " + "forge or alter a listing.", + "Every listing expires, can be disputed (it stops counting at once) and withdrawn; an " + "unanswered dispute is decided for the customer.", + "Every share, token request, feed and dispute is in a hash-chained audit log.", +) + + +def save_demo(out: Path, inp, market, run: dict, hub_key: crypto.OprfKey, report: dict) -> None: + hub: Hub = run["hub"] + members: dict[str, Member] = run["members"] + keys = { + "note": "simulated parties only; see consortium/demo.py", + "oprf": {"epoch": hub_key.epoch, "secret": f"{hub_key.secret:x}"}, + "members": {n: f"{m.key.secret:x}" for n, m in members.items()}, + } + (out / "demo_keys.json").write_text(json.dumps(keys)) + bundles = {n: b.to_dict() for n, b in hub.bundles.items()} + (out / "bundles.json").write_text(json.dumps(bundles)) + pd.DataFrame( + { + "wallet_id": list(market.provider), + "provider": list(market.provider.values()), + "msisdn": [market.msisdn[w] for w in market.provider], + "devices": [[d for _, d in inp.devices.get(w, ())] for w in market.provider], + } + ).to_parquet(out / "market.parquet", index=False) + (out / "hub_audit.json").write_text(json.dumps(hub.audit.entries)) + + # Test-period matches, one row per receiving wallet: its first partner match. + send, matches = run["send"], run["matches"] + is_t = (send["fold"] == "test").to_numpy() + truth = send[is_t].groupby("receiver_id")["y_mule"].max() + cell = inp.wallets.set_index("wallet_id")["cell_id"] + seen: dict[str, dict] = {} + for i in np.flatnonzero(is_t): + found = matches[i] + if not found: + continue + row = send.iloc[i] + wid = row["receiver_id"] + if wid in seen: + seen[wid]["transfers_matched"] += 1 + continue + seen[wid] = { + "wallet_id": wid, + "at": iso(datetime.fromtimestamp(float(row["t"]), tz=UTC)), + "home": market.provider[wid], + "partners": sorted({m.provider for m in found}), + "kinds": sorted({m.kind for m in found}), + "status": "confirmed" if any(m.status == "confirmed" for m in found) else "suspected", + "confidence": round(max(m.confidence for m in found), 3), + "typology": next((m.typology for m in found if m.status == "confirmed"), "unknown"), + "listing_ids": sorted({m.listing_id for m in found if m.listing_id}), + "mule_score": round(float(row["mule"]), 4), + "mule_model_alone_would_alert": bool(row["mule"] >= inp.threshold), + "transfers_matched": 1, + "simulation_truth": ( + "mule" + if truth.get(wid, 0) == 1 + else "ring wallet" + if cell.get(wid, -1) >= 0 + else "not a mule" + ), + } + (out / "matches.json").write_text(json.dumps(list(seen.values()))) + + +class ConsortiumService: + """The demo consortium, rebuilt from its saved state, as one process holds it.""" + + def __init__(self, directory: Path) -> None: + self.dir = directory + self.report = json.loads((directory / "report.json").read_text()) + keys = json.loads((directory / "demo_keys.json").read_text()) + bundles = { + n: Bundle.from_dict(b) + for n, b in json.loads((directory / "bundles.json").read_text()).items() + } + last = max(parse_iso(b.issued_at) for b in bundles.values()) + # The demo clock: an hour after the last bundles went out. + self.clock = last + timedelta(hours=1) + hub_key = crypto.OprfKey(keys["oprf"]["epoch"], int(keys["oprf"]["secret"], 16)) + self.hub = Hub(hub_key.epoch, last, key=hub_key) + self.hub.audit.entries = json.loads((directory / "hub_audit.json").read_text()) + self.chain_ok = self.hub.audit.verify() + displays = {p["name"]: p["display"] for p in self.report["providers"]} + self.members: dict[str, Member] = {} + for name, secret in keys["members"].items(): + key = crypto.SigningKey(int(secret, 16)) + self.hub.members[name] = _record(name, displays[name], key.public, last) + self.members[name] = Member(name, self.hub, key) + for name, bundle in bundles.items(): + if not bundle.verify(self.hub.members[name].public): + raise ProtocolError(f"saved bundle from {name} does not verify") + self.hub.bundles[name] = bundle + self.members[name].seq = bundle.seq + for m in self.members.values(): + m.sync(last) + market = pd.read_parquet(directory / "market.parquet") + self.provider = dict(zip(market["wallet_id"], market["provider"], strict=True)) + self.msisdn = dict(zip(market["wallet_id"], market["msisdn"], strict=True)) + self.devices = {w: list(d) for w, d in zip(market["wallet_id"], market["devices"], + strict=True)} # fmt: skip + self._matches = json.loads((directory / "matches.json").read_text()) + + @staticmethod + def load(artifacts_dir: Path) -> ConsortiumService | None: + directory = artifacts_dir / "consortium" + if not (directory / "demo_keys.json").is_file(): + return None + return ConsortiumService(directory) + + # -------------------------------------------------------------- views + + def overview(self) -> dict: + feeds = [] + for name, b in sorted(self.hub.bundles.items()): + feeds.append( + { + "provider": name, + "display": self.hub.members[name].display, + "seq": b.seq, + "issued_at": b.issued_at, + "valid_until": b.valid_until, + "confirmed_listings": len(b.listings), + "by_kind": dict(Counter(x.kind for x in b.listings)), + "by_typology": dict(Counter(x.typology for x in b.listings).most_common(6)), + "suspected_in_filter": b.watch["bloom"]["n"], + "filter_bits": b.watch["bloom"]["m"], + "filter_hashes": b.watch["bloom"]["k"], + "withdrawn": len(b.withdrawn), + "bytes": b.size_bytes(), + "signature_ok": b.verify(self.hub.members[name].public), + "key_fingerprint": crypto.fingerprint(self.hub.members[name].public), + } + ) + return { + "clock": iso(self.clock), + "key_epoch": self.hub.epoch, + "oprf_key_fingerprint": crypto.fingerprint(self.hub.public), + "members": [r.summary() for r in self.hub.members.values()], + "feeds": feeds, + "guarantees": list(GUARANTEES), + "audit": { + "entries": len(self.hub.audit.entries), + "chain_ok": self.hub.audit.verify(), + "head": self.hub.audit.head(), + }, # fmt: skip + "disputes": [d.to_dict() for d in self.hub.disputes.values()], + "results": self.report, + } + + def matches(self, limit: int = 200) -> list[dict]: + return self._matches[:limit] + + def audit(self, limit: int = 100) -> list[dict]: + return self.hub.audit.entries[-limit:][::-1] + + # ------------------------------------------------------------- actions + + def lookup(self, wallet_id: str) -> dict: + """What the wallet's own provider learns from partners about it, right now.""" + devices = self.devices.get(wallet_id, []) + home = self.provider.get(wallet_id) + if home is None: + raise ProtocolError("this wallet is not in the simulated market") + member = self.members[home] + ids = [("msisdn", self.msisdn[wallet_id])] + [("device", d) for d in devices] + score, found = member.lookup(ids, self.clock, purpose="analyst") + return { + "wallet_id": wallet_id, + "home": home, + "identifiers_checked": {"msisdn": 1, "device": len(devices)}, + "signal": round(score, 4), + "matches": [m.to_dict() for m in found], + "audit_head": member.audit.head(), + } + + def open_dispute(self, raised_by: str, listing_id: str, reason: str) -> dict: + dispute = self.hub.open_dispute(raised_by, listing_id, reason, self.clock) + for m in self.members.values(): + m.ledger.suspended.add(listing_id) + return dispute.to_dict() + + def resolve_dispute(self, dispute_id: str, outcome: str) -> dict: + dispute = self.hub.disputes.get(dispute_id) + if dispute is None: + raise ProtocolError(f"no dispute {dispute_id}") + # In the demo the console acts for the listing member. + self.hub.resolve_dispute(dispute_id, dispute.owner, outcome, self.clock) + if outcome == "upheld": + for m in self.members.values(): + m.ledger.suspended.discard(dispute.listing_id) + return dispute.to_dict() + + +def _record(name: str, display: str, public: int, at: datetime): + from .hub import MemberRecord + + return MemberRecord(name, display, public, iso(at)) diff --git a/backend/src/fraudlens/consortium/hub.py b/backend/src/fraudlens/consortium/hub.py new file mode 100644 index 0000000..154ba95 --- /dev/null +++ b/backend/src/fraudlens/consortium/hub.py @@ -0,0 +1,407 @@ +"""The two roles: the consortium hub and a member provider. + +The **hub** is run by the consortium (in Bangladesh, naturally under the central +bank's oversight). It holds the OPRF key, a register of members' signing keys, the +latest bundle from each member, and the dispute docket. It never sees a wallet +number: what reaches it is blinded group elements and already-tokenised bundles. + +A **member** is a provider. It keeps its raw mule list to itself, makes tokens by +asking the hub (blinded, rate-limited), publishes signed bundles, imports its +partners' bundles and looks up its own receivers locally. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass, field +from datetime import datetime, timedelta + +from . import crypto +from .protocol import ( + BUNDLE_TTL, + CONFIRMED_TTL, + SUSPECTED_TTL, + WATCH_FP_RATE, + AuditChain, + Bundle, + Ledger, + Listing, + Match, + ProtocolError, + group_element, + iso, + normalise, + parse_iso, + signal, + watch_key, +) + +DEFAULT_QUOTA = 250_000 # OPRF evaluations per member per day: onboarding plus daily traffic +DISPUTE_SLA = timedelta(days=5) # an unanswered dispute is decided for the customer +OUTCOMES = ("upheld", "withdrawn") + +Mapper = Callable[[Callable, Iterable], Iterable] + + +@dataclass +class MemberRecord: + name: str + display: str + public: int + joined_at: str + quota_per_day: int = DEFAULT_QUOTA + used: dict[str, int] = field(default_factory=dict) # day -> evaluations + disputes_against: int = 0 + withdrawn_after_dispute: int = 0 + + def summary(self) -> dict: + return { + "name": self.name, + "display": self.display, + "key_fingerprint": crypto.fingerprint(self.public), + "joined_at": self.joined_at, + "quota_per_day": self.quota_per_day, + "disputes_against": self.disputes_against, + "withdrawn_after_dispute": self.withdrawn_after_dispute, + } + + +@dataclass +class Dispute: + dispute_id: str + listing_id: str + owner: str + raised_by: str + reason: str + opened_at: str + status: str = "open" # open, upheld, withdrawn + resolved_at: str | None = None + resolved_by: str | None = None + + def to_dict(self) -> dict: + return asdict(self) + + +class Hub: + def __init__(self, epoch: str, now: datetime, key: crypto.OprfKey | None = None) -> None: + self.oprf = key or crypto.OprfKey.generate(epoch) + self.epoch = self.oprf.epoch + self.public = self.oprf.public + self.members: dict[str, MemberRecord] = {} + self.bundles: dict[str, Bundle] = {} + self.disputes: dict[str, Dispute] = {} + self.audit = AuditChain("hub") + self.audit.append(now, "hub.started", key_epoch=self.epoch, + oprf_key=crypto.fingerprint(self.public)) # fmt: skip + + # -------------------------------------------------------------- members + + def register(self, name: str, display: str, public: int, now: datetime, **kw) -> None: + if name in self.members: + raise ProtocolError(f"{name} is already a member") + self.members[name] = MemberRecord(name, display, public, iso(now), **kw) + self.audit.append(now, "member.joined", member=name, + signing_key=crypto.fingerprint(public)) # fmt: skip + + def _member(self, name: str) -> MemberRecord: + record = self.members.get(name) + if record is None: + raise ProtocolError(f"{name} is not a member") + return record + + # ----------------------------------------------------------------- OPRF + + def evaluate( + self, member: str, blinded: list[int], now: datetime, mapper: Mapper = map + ) -> list[int]: + """Raise blinded elements to the key, within the member's daily quota. + + The quota is what stops a member from tokenising the whole numbering plan + to reverse partners' tokens: 10^9 numbers at this quota is 10,000 years. + """ + record = self._member(member) + day = now.strftime("%Y-%m-%d") + used = record.used.get(day, 0) + if used + len(blinded) > record.quota_per_day: + self.audit.append(now, "oprf.refused", member=member, requested=len(blinded), + used_today=used) # fmt: skip + raise ProtocolError("daily token quota exceeded") + record.used[day] = used + len(blinded) + out = list(mapper(self.oprf.evaluate, blinded)) + self.audit.append(now, "oprf.evaluated", member=member, count=len(blinded)) + return out + + # -------------------------------------------------------------- bundles + + def submit(self, bundle: Bundle, now: datetime) -> dict: + record = self._member(bundle.provider) + if not bundle.verify(record.public): + self.audit.append(now, "share.rejected", member=bundle.provider, why="signature") + raise ProtocolError("bundle signature does not verify") + if bundle.key_epoch != self.epoch: + raise ProtocolError("bundle tokens are from another key epoch") + held = self.bundles.get(bundle.provider) + if held is not None and bundle.seq <= held.seq: + raise ProtocolError("bundle is not newer than the one held") + self.bundles[bundle.provider] = bundle + receipt = { + "member": bundle.provider, + "seq": bundle.seq, + "listings": len(bundle.listings), + "watch": bundle.watch["bloom"]["n"], + "withdrawn": len(bundle.withdrawn), + "bytes": bundle.size_bytes(), + } + self.audit.append(now, "share.accepted", **receipt) + return receipt + + def feed(self, member: str, now: datetime) -> dict: + """Everything a member needs to refresh its ledger.""" + self._member(member) + others = [b for name, b in sorted(self.bundles.items()) if name != member] + self.audit.append(now, "feed.served", member=member, + bundles=[f"{b.provider}#{b.seq}" for b in others]) # fmt: skip + return { + "key_epoch": self.epoch, + "bundles": others, + "keys": {name: r.public for name, r in self.members.items()}, + "suspended": sorted(d.listing_id for d in self.disputes.values() if d.status == "open"), + "removed": sorted( + d.listing_id for d in self.disputes.values() if d.status == "withdrawn" + ), + } + + # ------------------------------------------------------------- disputes + + def open_dispute(self, raised_by: str, listing_id: str, reason: str, now: datetime) -> Dispute: + """Contest a listing. It stops counting at once and stays out until decided. + + Anyone with standing can raise one: a partner whose customer was matched, + or the listing member itself on behalf of its customer's appeal. + """ + self._member(raised_by) + owner = listing_id.split(":", 1)[0] + bundle = self.bundles.get(owner) + if bundle is None or not any(x.listing_id == listing_id for x in bundle.listings): + raise ProtocolError(f"no current listing {listing_id}") + if any(d.listing_id == listing_id and d.status == "open" for d in self.disputes.values()): + raise ProtocolError(f"{listing_id} is already under dispute") + if not 3 <= len(reason.strip()) <= 500: + raise ProtocolError("a dispute needs a reason of 3 to 500 characters") + dispute = Dispute(f"D{len(self.disputes) + 1:04d}", listing_id, owner, raised_by, + reason.strip(), iso(now)) # fmt: skip + self.disputes[dispute.dispute_id] = dispute + self._member(owner).disputes_against += 1 + self.audit.append(now, "dispute.opened", dispute=dispute.dispute_id, listing=listing_id, + owner=owner, raised_by=raised_by) # fmt: skip + return dispute + + def resolve_dispute(self, dispute_id: str, by: str, outcome: str, now: datetime) -> Dispute: + """Only the listing member decides, because only it holds the evidence.""" + dispute = self.disputes.get(dispute_id) + if dispute is None: + raise ProtocolError(f"no dispute {dispute_id}") + if dispute.status != "open": + raise ProtocolError(f"{dispute_id} is already {dispute.status}") + if by not in (dispute.owner, "hub"): + raise ProtocolError("only the listing member (or the hub, on expiry) can decide") + if outcome not in OUTCOMES: + raise ProtocolError(f"outcome must be one of {OUTCOMES}") + dispute.status, dispute.resolved_at, dispute.resolved_by = outcome, iso(now), by + if outcome == "withdrawn": + self._member(dispute.owner).withdrawn_after_dispute += 1 + self.audit.append(now, "dispute.resolved", dispute=dispute_id, + listing=dispute.listing_id, outcome=outcome, by=by) # fmt: skip + return dispute + + def expire_disputes(self, now: datetime) -> list[Dispute]: + """Disputes the listing member did not answer in time are decided for the customer.""" + late = [ + d + for d in self.disputes.values() + if d.status == "open" and parse_iso(d.opened_at) + DISPUTE_SLA <= now + ] + return [self.resolve_dispute(d.dispute_id, "hub", "withdrawn", now) for d in late] + + +@dataclass +class _Local: + """A member's own record of a listed identifier. The raw value never leaves it.""" + + listing_id: str + kind: str + value: str + status: str # confirmed or suspected + confidence: float + typology: str + first_seen: datetime + listed_at: datetime + expires_at: datetime + + +class Member: + def __init__(self, name: str, hub: Hub, key: crypto.SigningKey | None = None) -> None: + self.name = name + self.hub = hub + self.key = key or crypto.SigningKey.generate() + self.ledger = Ledger(name) + self.audit = AuditChain(name) + self.seq = 0 + self._next_id = 0 + self._local: dict[tuple[str, str], _Local] = {} + self._tokens: dict[tuple[str, str], str] = {} # own identifiers, this key epoch + self._withdrawn: list[str] = [] + + # ------------------------------------------------------------ tokens + + def tokenize( + self, pairs: list[tuple[str, str]], now: datetime, mapper: Mapper = map + ) -> list[str]: + """Tokens for (kind, value) pairs, through the hub's blinded OPRF. + + Tokens of the member's own customers are cached for the key epoch, so a + payment-time lookup normally needs no hub round trip at all. + """ + keys = [(kind, normalise(kind, value)) for kind, value in pairs] + missing = sorted({k for k in keys if k not in self._tokens}) + if missing: + elements = [group_element(kind, value) for kind, value in missing] + blinded = list(mapper(crypto.blind, elements)) + evaluated = self.hub.evaluate(self.name, [b for b, _ in blinded], now, mapper) + public = self.hub.public + unblinded = mapper(_unblind, [(e, r, public) for e, (_, r) in + zip(evaluated, blinded, strict=True)]) # fmt: skip + for (kind, value), element_k in zip(missing, unblinded, strict=True): + self._tokens[kind, value] = crypto.token_from_element( + element_k, self.hub.epoch, kind + ) + return [self._tokens[k] for k in keys] + + # ----------------------------------------------------------- listings + + def list_wallet( + self, + identifiers: list[tuple[str, str]], + status: str, + confidence: float, + now: datetime, + typology: str = "unknown", + first_seen: datetime | None = None, + ) -> list[str]: + """Put a wallet's identifiers on the local list as confirmed or suspected.""" + if status not in ("confirmed", "suspected"): + raise ProtocolError("status must be confirmed or suspected") + ttl = CONFIRMED_TTL if status == "confirmed" else SUSPECTED_TTL + ids = [] + for kind, value in identifiers: + key = (kind, normalise(kind, value)) + held = self._local.get(key) + if held is not None and held.status == "confirmed" and status == "suspected": + ids.append(held.listing_id) # never downgrade a confirmed listing + continue + if held is not None and held.status == status: + held.expires_at = now + ttl + held.confidence = max(held.confidence, confidence) + ids.append(held.listing_id) + continue + self._next_id += 1 + listing_id = f"{self.name}:{self._next_id}" + self._local[key] = _Local(listing_id, kind, key[1], status, confidence, typology, + first_seen or now, now, now + ttl) # fmt: skip + ids.append(listing_id) + return ids + + def withdraw(self, listing_id: str, now: datetime, why: str) -> None: + """Take a listing back: it is removed locally and named in the next bundle.""" + for key, item in list(self._local.items()): + if item.listing_id == listing_id: + del self._local[key] + self._withdrawn.append(listing_id) + self.audit.append(now, "listing.withdrawn", listing=listing_id, why=why) + return + raise ProtocolError(f"{listing_id} is not one of this member's listings") + + def export_bundle(self, now: datetime, watch_confidence: float, mapper: Mapper = map) -> Bundle: + """A signed snapshot of the active list, tokenised. Nothing raw goes in it.""" + active = [x for x in self._local.values() if x.listed_at <= now < x.expires_at] + tokens = self.tokenize([(x.kind, x.value) for x in active], now, mapper) + listings, watch = [], [] + for item, token in zip(active, tokens, strict=True): + if item.status == "confirmed": + listings.append( + Listing( + item.listing_id, + token, + item.kind, + round(item.confidence, 3), + item.typology, + iso(item.first_seen), + iso(item.listed_at), + iso(item.expires_at), + ) # fmt: skip + ) + else: + watch.append(watch_key(item.kind, token)) + bloom = crypto.BloomFilter.sized(len(watch), WATCH_FP_RATE) + for key in watch: + bloom.add(key) + self.seq += 1 + bundle = Bundle( + provider=self.name, + seq=self.seq, + key_epoch=self.hub.epoch, + issued_at=iso(now), + valid_until=iso(now + BUNDLE_TTL), + listings=sorted(listings, key=lambda x: x.listing_id), + watch={"confidence": round(watch_confidence, 3), "bloom": bloom.to_dict()}, + withdrawn=list(self._withdrawn), + ).sign(self.key) + self.audit.append(now, "share.exported", seq=bundle.seq, listings=len(listings), + watch=len(watch), withdrawn=len(self._withdrawn)) # fmt: skip + return bundle + + def publish(self, now: datetime, watch_confidence: float, mapper: Mapper = map) -> dict: + return self.hub.submit(self.export_bundle(now, watch_confidence, mapper), now) + + # ----------------------------------------------------------- partners + + def sync(self, now: datetime) -> list[dict]: + """Import every partner bundle the hub holds that is newer than ours.""" + feed = self.hub.feed(self.name, now) + imported = [] + for bundle in feed["bundles"]: + held = self.ledger.bundles.get(bundle.provider) + if held is not None and held.seq >= bundle.seq: + continue + receipt = self.ledger.import_bundle( + bundle, feed["keys"][bundle.provider], feed["key_epoch"], now + ) + self.audit.append(now, "share.imported", **receipt) + imported.append(receipt) + self.ledger.suspended = set(feed["suspended"]) | set(feed["removed"]) + # A dispute decided against one of our listings: take it off our own list too. + own = {x.listing_id for x in self._local.values()} + for listing_id in feed["removed"]: + if listing_id in own: + self.withdraw(listing_id, now, "dispute decided for the customer") + return imported + + def lookup( + self, identifiers: list[tuple[str, str]], at: datetime, purpose: str = "payment" + ) -> tuple[float, list[Match]]: + """Receiver-side check: what partners know about these identifiers. + + Audited with the count of identifiers and of matches, never the values. + """ + tokens = self.tokenize(identifiers, at) + matches = self.ledger.lookup( + [(kind, t) for (kind, _), t in zip(identifiers, tokens, strict=True)], at + ) + self.audit.append(at, "lookup", purpose=purpose, identifiers=len(identifiers), + matches=len(matches), + listings=[m.listing_id for m in matches if m.listing_id]) # fmt: skip + return signal(matches), matches + + +def _unblind(args: tuple[int, int, int]) -> int: + return crypto.unblind(*args) diff --git a/backend/src/fraudlens/consortium/protocol.py b/backend/src/fraudlens/consortium/protocol.py new file mode 100644 index 0000000..1c7573c --- /dev/null +++ b/backend/src/fraudlens/consortium/protocol.py @@ -0,0 +1,321 @@ +"""The consortium protocol: what a provider shares, how a partner checks it, how it is undone. + +A provider (a *member*) keeps its mule list locally, with real wallet numbers and +handset fingerprints. What leaves it is a **bundle**, signed by the member: + +- `listings`: confirmed mules, one entry per identifier token, each with a + confidence, a typology, when the provider first saw it and when it expires; +- `watch`: suspected (not yet confirmed) wallets as a Bloom filter of tokens, + which a partner can test a token against but cannot enumerate; +- `withdrawn`: listings taken back since the last bundle (disputes upheld for the + customer, or the member's own correction). + +Tokens are made through the hub's oblivious PRF (see `crypto`), so a bundle holds no +wallet number, no handset id and nothing a partner can reverse. A partner imports +bundles into its **ledger** and looks up its own receivers at payment time: a local +dictionary and Bloom probe, no network call on the payment path. + +Every share, import, lookup and dispute step is appended to a hash-chained +**audit log**; changing or removing any past line breaks the chain. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime, timedelta + +from . import crypto + +FORMAT = "fraudlens-consortium-bundle/1" +KINDS = ("msisdn", "device") +TYPOLOGIES_UNKNOWN = "unknown" +CONFIRMED_TTL = timedelta(days=180) +SUSPECTED_TTL = timedelta(days=30) +BUNDLE_TTL = timedelta(days=2) # a partner stops trusting a feed it has not refreshed +WATCH_FP_RATE = 1e-6 # per partner, per lookup: lookups run at every transfer +MAX_LISTINGS = 200_000 + +_MSISDN = re.compile(r"^(?:\+?88)?(01[3-9]\d{8})$") + + +class ProtocolError(ValueError): + """A bundle, lookup or dispute that the protocol refuses.""" + + +def iso(ts: datetime) -> str: + return ts.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_iso(value: str) -> datetime: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC) + + +def normalise(kind: str, value: str) -> str: + """The one spelling of an identifier that every member hashes. + + A Bangladeshi wallet number is the owner's mobile number: 01XXXXXXXXX, with or + without +88. It is normalised to 8801XXXXXXXXX so that every provider derives + the same token for the same SIM. + """ + if kind == "msisdn": + match = _MSISDN.match(re.sub(r"[\s-]", "", value)) + if not match: + raise ProtocolError("not a Bangladeshi mobile number") + return "88" + match.group(1) + if kind == "device": + value = value.strip().lower() + if not 4 <= len(value) <= 128: + raise ProtocolError("device fingerprint must be 4 to 128 characters") + return value + raise ProtocolError(f"unknown identifier kind {kind!r}") + + +def group_element(kind: str, value: str) -> int: + return crypto.hash_to_group(f"{kind}:{normalise(kind, value)}".encode()) + + +def canonical(obj: dict) -> bytes: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + + +# ------------------------------------------------------------------ listings + + +@dataclass(frozen=True) +class Listing: + listing_id: str # ":", stable for the life of the listing + token: str + kind: str + confidence: float + typology: str + first_seen: str + listed_at: str + expires_at: str + + def active(self, at: datetime) -> bool: + return parse_iso(self.listed_at) <= at < parse_iso(self.expires_at) + + +@dataclass +class Bundle: + provider: str + seq: int + key_epoch: str + issued_at: str + valid_until: str + listings: list[Listing] + watch: dict # {"confidence": float, "bloom": BloomFilter.to_dict()} + withdrawn: list[str] + signature: str = "" + format: str = FORMAT + + def body(self) -> dict: + d = asdict(self) + d.pop("signature") + return d + + def to_dict(self) -> dict: + return asdict(self) + + @staticmethod + def from_dict(d: dict) -> Bundle: + if d.get("format") != FORMAT: + raise ProtocolError("unknown bundle format") + if len(d["listings"]) > MAX_LISTINGS: + raise ProtocolError("bundle is larger than the protocol allows") + listings = [Listing(**item) for item in d["listings"]] + for item in listings: + if item.kind not in KINDS or not 0.0 <= item.confidence <= 1.0: + raise ProtocolError(f"malformed listing {item.listing_id}") + if not re.fullmatch(r"[0-9a-f]{32}", item.token): + raise ProtocolError(f"listing {item.listing_id} does not carry a token") + return Bundle( + provider=d["provider"], + seq=int(d["seq"]), + key_epoch=d["key_epoch"], + issued_at=d["issued_at"], + valid_until=d["valid_until"], + listings=listings, + watch=d["watch"], + withdrawn=list(d["withdrawn"]), + signature=d["signature"], + ) + + def sign(self, key: crypto.SigningKey) -> Bundle: + self.signature = key.sign(canonical(self.body())) + return self + + def verify(self, public: int) -> bool: + return crypto.verify(public, canonical(self.body()), self.signature) + + def size_bytes(self) -> int: + return len(canonical(self.to_dict())) + + +# --------------------------------------------------------------------- audit + + +@dataclass +class AuditChain: + """Append-only log where each line commits to the one before it.""" + + owner: str + entries: list[dict] = field(default_factory=list) + + GENESIS = "0" * 64 + + def append(self, at: datetime, event: str, **detail) -> dict: + prev = self.entries[-1]["hash"] if self.entries else self.GENESIS + line = { + "n": len(self.entries) + 1, + "at": iso(at), + "by": self.owner, + "event": event, + "detail": detail, + "prev": prev, + } + line["hash"] = hashlib.sha256(canonical(line)).hexdigest() + self.entries.append(line) + return line + + def verify(self) -> bool: + prev = self.GENESIS + for i, line in enumerate(self.entries, start=1): + body = {k: v for k, v in line.items() if k != "hash"} + if line["n"] != i or line["prev"] != prev: + return False + if hashlib.sha256(canonical(body)).hexdigest() != line["hash"]: + return False + prev = line["hash"] + return True + + def head(self) -> str: + return self.entries[-1]["hash"] if self.entries else self.GENESIS + + +# -------------------------------------------------------------------- ledger + + +@dataclass(frozen=True) +class Match: + provider: str + kind: str + status: str # "confirmed" or "suspected" + confidence: float + typology: str + listing_id: str | None # None for a watch-list (Bloom) hit + first_seen: str | None + + def to_dict(self) -> dict: + return asdict(self) + + +def signal(matches: list[Match]) -> float: + """One receiver-side number from any number of partner matches. + + Each listing is treated as independent evidence of its stated confidence; a + provider listing the same wallet under two identifiers counts once, at its + strongest. 0 means no partner knows anything about this wallet. + """ + best: dict[str, float] = {} + for m in matches: + best[m.provider] = max(best.get(m.provider, 0.0), m.confidence) + miss = 1.0 + for c in best.values(): + miss *= 1.0 - c + return 1.0 - miss + + +class Ledger: + """One member's view of every partner's latest bundle.""" + + def __init__(self, owner: str) -> None: + self.owner = owner + self.bundles: dict[str, Bundle] = {} + self._by_token: dict[str, list[Listing]] = {} + self._watch: dict[str, tuple[crypto.BloomFilter, float]] = {} + self.suspended: set[str] = set() # listings under dispute: not used while it runs + + def import_bundle(self, bundle: Bundle, public: int, key_epoch: str, now: datetime) -> dict: + """Check a partner bundle and make it the partner's current one.""" + if bundle.provider == self.owner: + raise ProtocolError("a member does not import its own bundle") + if not bundle.verify(public): + raise ProtocolError(f"bad signature on the bundle from {bundle.provider}") + if bundle.key_epoch != key_epoch: + raise ProtocolError("bundle tokens are from another key epoch") + current = self.bundles.get(bundle.provider) + if current is not None and bundle.seq <= current.seq: + raise ProtocolError("bundle is not newer than the one held (replay)") + if parse_iso(bundle.valid_until) <= now: + raise ProtocolError("bundle has expired") + self.bundles[bundle.provider] = bundle + self._reindex() + return { + "provider": bundle.provider, + "seq": bundle.seq, + "listings": len(bundle.listings), + "watch": bundle.watch["bloom"]["n"], + "withdrawn": len(bundle.withdrawn), + } + + def _reindex(self) -> None: + self._by_token = {} + self._watch = {} + for provider, bundle in self.bundles.items(): + gone = set(bundle.withdrawn) + for item in bundle.listings: + if item.listing_id not in gone: + self._by_token.setdefault(item.token, []).append(item) + bloom = crypto.BloomFilter.from_dict(bundle.watch["bloom"]) + self._watch[provider] = (bloom, float(bundle.watch["confidence"])) + + def lookup(self, tokens: list[tuple[str, str]], at: datetime) -> list[Match]: + """Partner knowledge about these (kind, token) pairs at time `at`.""" + out: list[Match] = [] + for kind, token in tokens: + for item in self._by_token.get(token, ()): + provider = item.listing_id.split(":", 1)[0] + if ( + item.kind == kind + and item.active(at) + and item.listing_id not in self.suspended + and parse_iso(self.bundles[provider].valid_until) > at + ): + out.append( + Match( + provider, + kind, + "confirmed", + item.confidence, + item.typology, + item.listing_id, + item.first_seen, + ) # fmt: skip + ) + for provider, (bloom, confidence) in self._watch.items(): + if ( + parse_iso(self.bundles[provider].valid_until) > at + and f"{kind}:{token}" in bloom + ): + out.append( + Match( + provider, kind, "suspected", confidence, TYPOLOGIES_UNKNOWN, None, None + ) # fmt: skip + ) + return out + + def listing(self, listing_id: str) -> Listing | None: + provider = listing_id.split(":", 1)[0] + bundle = self.bundles.get(provider) + if bundle is None: + return None + return next((x for x in bundle.listings if x.listing_id == listing_id), None) + + +def watch_key(kind: str, token: str) -> str: + """What goes into a watch filter: the token bound to its kind.""" + return f"{kind}:{token}" diff --git a/backend/src/fraudlens/consortium/simulate.py b/backend/src/fraudlens/consortium/simulate.py new file mode 100644 index 0000000..766e313 --- /dev/null +++ b/backend/src/fraudlens/consortium/simulate.py @@ -0,0 +1,627 @@ +"""A four-provider market on top of the existing dataset, and the consortium experiment. + + uv run python -m fraudlens.consortium.simulate # backend/data/full + uv run python -m fraudlens.consortium.simulate --quick # one seed, for a smoke run + +Opt-in and read-only towards everything else: the dataset, the features and the +model versions are used as they are, and the output goes to +`artifacts/consortium/` only. + +**The market.** Every wallet of the dataset is given to one of four simulated +providers, and a synthetic wallet number (MSISDN). A share of people hold wallets at +two providers on one SIM; this is applied to mule-cell wallets and to ordinary +customers at the *same* rate, so the experiment does not assume mules multi-home +more than anyone else. Handsets come from the dataset itself: a cell's handset used +by wallets that ended up at different providers is a real cross-provider link. + +**The experiment.** Each provider scores receivers with the served mule model, as +today. Once a day it lists its confirmed mules (victim-reported) and its suspected +ones (mule-model alerts) in a signed bundle, through the real protocol in +`hub.py`. At every transfer the receiving provider looks the receiver up in its +partners' bundles. A receiver is alerted when the mule score clears its bar *or* +the consortium signal clears its own. Both bars are chosen on `val_b` (the mule bar +can only rise from the served threshold) so that no more non-mule wallets are +alerted there than with the mule model alone. Everything reported is measured on the +untouched test period. +""" + +from __future__ import annotations + +import argparse +import json +import time +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pandas as pd + +from ..config import settings +from ..features import RECIPIENT_FEATURES, FeatureEngine +from ..models import registry +from ..models.data import load_frame +from ..models.rings import MAX_DEVICE_WALLETS +from . import crypto +from .hub import Hub, Member +from .protocol import SUSPECTED_TTL, iso, signal + +PROVIDERS: tuple[tuple[str, str, float], ...] = ( + # Illustrative shares for the simulation, not market data. + ("bkash", "bKash (simulated)", 0.55), + ("nagad", "Nagad (simulated)", 0.25), + ("rocket", "Rocket (simulated)", 0.12), + ("upay", "upay (simulated)", 0.08), +) +KEY_EPOCH = "2026Q1" +SEEDS = (11, 12, 13, 14, 15) +SIM_SHARES = (0.0, 0.25, 0.5) +HEADLINE_SHARE = 0.25 +ARMS = {"mule_model_only": (), "device": ("device",), "msisdn": ("msisdn",), + "msisdn_and_device": ("msisdn", "device")} # fmt: skip +# A rule fixed in advance instead of tuned on val_b: alert on any receiver that a +# partner has listed as a *confirmed* mule (signal >= the confirmed confidence), +# on top of the served mule model at its served threshold. +RULE_ARMS = {"msisdn_confirmed_rule": ("msisdn",), + "msisdn_and_device_confirmed_rule": ("msisdn", "device")} # fmt: skip +ALL_ARMS = {**ARMS, **RULE_ARMS} +POISON_LISTINGS = 500 + + +def out_dir() -> Path: + return settings.artifacts_dir / "consortium" + + +def _epoch(series: pd.Series) -> np.ndarray: + return series.astype("datetime64[us]").astype("int64").to_numpy() / 1e6 + + +def _dt(ts: float) -> datetime: + return datetime.fromtimestamp(ts, UTC) + + +# ------------------------------------------------------------------ the market + + +@dataclass +class Market: + provider: dict[str, str] # wallet -> provider + msisdn: dict[str, str] # wallet -> synthetic wallet number + shared_sim: dict[str, str] # wallet -> the wallet it shares a SIM with + + def cell_spread(self, cells: pd.Series) -> dict: + """How many providers each scam cell's wallets ended up spread over.""" + spread = cells.groupby(cells.values).apply( + lambda ws: len({self.provider[w] for w in ws.index}) + ) + return { + "cells": int(len(spread)), + "on_one_provider": int((spread == 1).sum()), + "on_two_or_more": int((spread >= 2).sum()), + "on_all_four": int((spread == 4).sum()), + } + + +def build_market(wallets: pd.DataFrame, seed: int, sim_share: float) -> Market: + rng = np.random.default_rng(seed) + names = [p[0] for p in PROVIDERS] + shares = np.array([p[2] for p in PROVIDERS]) + ids = wallets["wallet_id"].sort_values().tolist() + provider = dict(zip(ids, rng.choice(names, size=len(ids), p=shares), strict=True)) + + numbers = set() + msisdn = {} + for w in ids: + while True: + n = f"01{rng.integers(3, 10)}{rng.integers(0, 10**8):08d}" + if n not in numbers: + numbers.add(n) + msisdn[w] = n + break + + shared: dict[str, str] = {} + cell = wallets.set_index("wallet_id")["cell_id"] + groups = [g.index.tolist() for _, g in cell[cell >= 0].groupby(cell[cell >= 0])] + groups.append(cell[cell < 0].index.tolist()) # ordinary customers, at the same rate + for group in groups: + order = rng.permutation(sorted(group)).tolist() + for a, b in zip(order[0::2], order[1::2], strict=False): + if rng.random() >= sim_share: + continue + # One SIM holds at most one wallet per provider. + if provider[b] == provider[a]: + others = [n for n in names if n != provider[a]] + weights = np.array([s for n, s in zip(names, shares, strict=True) + if n != provider[a]]) # fmt: skip + provider[b] = str(rng.choice(others, p=weights / weights.sum())) + msisdn[b] = msisdn[a] + shared[a], shared[b] = b, a + return Market(provider, msisdn, shared) + + +# ---------------------------------------------------------------- the inputs + + +@dataclass +class Inputs: + send: pd.DataFrame # val_a, val_b and test transfers with the mule score + wallets: pd.DataFrame + devices: dict[str, list[tuple[float, str]]] # wallet -> [(first seen, device)] + typology: dict[str, str] + threshold: float + model_version: str + folds: dict[str, tuple[float, float]] # fold -> [start, end) epoch seconds + + +def load_inputs(data_dir: Path) -> Inputs: + frame = load_frame(data_dir) + frame = frame[~frame["ambiguous"] & (frame["type"] == "SEND_MONEY")] + frame = frame[frame["fold"].isin(("val_a", "val_b", "test"))].copy() + version = registry.current_version() + bundle = registry.load(version) + frame["mule"] = bundle.recipient_risk(frame[list(RECIPIENT_FEATURES)].to_numpy()) + frame["t"] = _epoch(frame["ts"]) + send = frame[["txn_id", "t", "day", "fold", "receiver_id", "amount", "mule", "y_mule", + "y_loss"]].sort_values(["t", "txn_id"]).reset_index(drop=True) # fmt: skip + + wallets = pd.read_parquet(data_dir / "wallets.parquet") + engine = FeatureEngine.load(data_dir / "engine_after_test.pkl") + devices: dict[str, list[tuple[float, str]]] = {} + for wid, state in engine.wallets.items(): + handsets = [ + (ts, d) + for d, ts in state.devices.items() + if len(engine.device_wallets.get(d, ())) <= MAX_DEVICE_WALLETS + ] + if handsets: + devices[wid] = sorted(handsets) + cases = pd.read_parquet(data_dir / "cases.parquet", columns=["mule_id", "typology"]) + typology = cases.groupby("mule_id")["typology"].agg(lambda s: s.mode().iloc[0]).to_dict() + cfg = json.loads((data_dir / "meta.json").read_text())["config"] + start = datetime.fromisoformat(cfg["start"]).replace(tzinfo=UTC).timestamp() + val_mid = (cfg["train_end_day"] + cfg["val_end_day"]) // 2 + days = {"val_a": (cfg["train_end_day"], val_mid), "val_b": (val_mid, cfg["val_end_day"]), + "test": (cfg["val_end_day"], cfg["days"])} # fmt: skip + folds = {k: (start + a * 86_400.0, start + b * 86_400.0) for k, (a, b) in days.items()} + return Inputs(send, wallets, devices, typology, + float(bundle.manifest["mule_wallet_threshold"]), version, folds) # fmt: skip + + +# ------------------------------------------------------------- measured priors + + +def measured_confidences(inp: Inputs) -> dict: + """What a listing is worth, measured before the test period starts. + + Confirmed: the share of wallets reported by victims before the test period + that really received fraud money. Suspected: the wallet-level precision of the + mule model's alerts on val_b. + """ + test_start = inp.folds["test"][0] + w = inp.wallets + flagged = w[w["flagged_at"].notna() & (_epoch(w["flagged_at"].fillna(pd.Timestamp(0))) + < test_start)] # fmt: skip + confirmed = float(flagged["first_fraud_at"].notna().mean()) + vb = inp.send[inp.send["fold"] == "val_b"] + by = vb.groupby("receiver_id").agg(score=("mule", "max"), y=("y_mule", "max")) + alerted = by[by["score"] >= inp.threshold] + suspected = float(alerted["y"].mean()) + return { + "confirmed": round(confirmed, 3), + "confirmed_basis": f"{len(flagged)} wallets reported before the test period", + "suspected": round(suspected, 3), + "suspected_basis": f"{len(alerted)} wallets alerted by the mule model on val_b", + } + + +# -------------------------------------------------------------------- the run + + +def _identifiers(inp: Inputs, market: Market, wallet: str, at: float) -> list[tuple[str, str]]: + ids = [("msisdn", market.msisdn[wallet])] + ids += [("device", d) for ts, d in inp.devices.get(wallet, ()) if ts <= at] + return ids + + +def run_consortium( + inp: Inputs, + market: Market, + conf: dict, + hub_key: crypto.OprfKey, + token_cache: dict, + mapper=map, + poison: list[str] | None = None, + keep: bool = False, +) -> dict: + """Day by day: list, publish, import, then look up every transfer of that day. + + Returns the consortium matches for every val_b and test transfer, and (with + `keep`) the hub and members for the demo state. + """ + t_begin, t_end = inp.folds["val_b"][0], inp.folds["test"][1] + hub = Hub(KEY_EPOCH, _dt(inp.folds["val_a"][0]), key=hub_key) + members: dict[str, Member] = {} + for name, display, _ in PROVIDERS: + m = Member(name, hub) + m._tokens = token_cache # see main(): every member's own tokens, made once + hub.register(name, display, m.key.public, _dt(inp.folds["val_a"][0])) + members[name] = m + + w = inp.wallets.set_index("wallet_id") + flagged_at = _epoch(w["flagged_at"].fillna(pd.Timestamp(0))) + flagged = {wid: ts for wid, ts, ok in zip(w.index, flagged_at, w["flagged_at"].notna(), + strict=True) if ok} # fmt: skip + crossings = inp.send[inp.send["mule"] >= inp.threshold] + crossed = list(zip(crossings["t"], crossings["receiver_id"], crossings["mule"], strict=True)) + + send = inp.send[(inp.send["t"] >= t_begin) & (inp.send["t"] < t_end)] + matches: list[list] = [[] for _ in range(len(send))] + i_cross, prev = 0, t_begin - SUSPECTED_TTL.total_seconds() + rows = list(zip(send["t"], send["receiver_id"], strict=True)) + i_row = 0 + day = t_begin + while day < t_end: + now = _dt(day) + # Confirmed: reported by victims, listed with every handset seen so far. + for wid, ts in flagged.items(): + if ts <= day: + p = market.provider[wid] + members[p].list_wallet( + _identifiers(inp, market, wid, day), + "confirmed", + conf["confirmed"], + now, + typology=inp.typology.get(wid, "unknown"), + first_seen=_dt(ts), + ) + # Suspected: the provider's own mule model alerted since the last bundle. + while i_cross < len(crossed) and crossed[i_cross][0] <= day: + ts, wid, _score = crossed[i_cross] + i_cross += 1 + if ts > prev: + p = market.provider[wid] + members[p].list_wallet(_identifiers(inp, market, wid, ts), "suspected", + conf["suspected"], _dt(ts)) # fmt: skip + if poison and day == inp.folds["test"][0]: + # A dishonest member lists innocent customers' numbers as confirmed mules. + members["upay"].list_wallet([("msisdn", market.msisdn[v]) for v in poison], + "confirmed", conf["confirmed"], now) # fmt: skip + prev = day + for m in members.values(): + m.publish(now, conf["suspected"], mapper) + for m in members.values(): + m.sync(now) + nxt = day + 86_400.0 + while i_row < len(rows) and rows[i_row][0] < nxt: + t, wid = rows[i_row] + home = members[market.provider[wid]] + ids = _identifiers(inp, market, wid, t) + tokens = home.tokenize(ids, _dt(t)) + found = home.ledger.lookup( + [(k, tok) for (k, _), tok in zip(ids, tokens, strict=True)], _dt(t) + ) + matches[i_row] = found + i_row += 1 + day = nxt + out = {"send": send.reset_index(drop=True), "matches": matches} + if keep: + out.update(hub=hub, members=members) + return out + + +# ----------------------------------------------------------------- evaluation + + +def _wallet_scores(send: pd.DataFrame, score: np.ndarray) -> pd.DataFrame: + return ( + send.assign(score=score) + .groupby("receiver_id") + .agg(score=("score", "max"), y=("y_mule", "max")) + ) + + +def _fused(base: np.ndarray, c: np.ndarray, t_mule: float, t_cons: float) -> np.ndarray: + """Alert when the mule model or the consortium signal clears its own bar (>= 1).""" + return np.maximum(base / t_mule, c / t_cons) + + +def _choose_thresholds( + send_b: pd.DataFrame, base: np.ndarray, c: np.ndarray, t0: float, allowed_fp: int +) -> tuple[float, float]: + """On val_b only: the (mule, consortium) bars that catch the most mule wallets + while alerting no more non-mule wallets than the mule model alone does. + + The mule bar can only go up from the served threshold; the consortium bar is + one of the signal levels actually seen (or off). Ties keep the served bar. + """ + by_base = _wallet_scores(send_b, base) + above = by_base.loc[by_base["score"] >= t0, "score"].to_numpy() + t_mules = sorted({t0, *np.quantile(above, np.linspace(0, 1, 21)).tolist()} if len(above) + else {t0}) # fmt: skip + t_conss = sorted({float(v) for v in np.unique(np.round(c[c > 0], 6))}) + [np.inf] + best = (-1, 0, np.inf, t0, np.inf) + for tm in t_mules: + for tc in t_conss: + by = _wallet_scores(send_b, _fused(base, c, tm, tc)) + alerted = by["score"] >= 1.0 + fp = int((alerted & (by["y"] == 0)).sum()) + tp = int((alerted & (by["y"] == 1)).sum()) + key = (tp, -(tm - t0), -fp) + if fp <= allowed_fp and key > best[:3]: + best = (*key, tm, tc) + return best[3], best[4] + + +def _timing(send: pd.DataFrame, score: np.ndarray, thr: float, flagged: dict) -> dict: + """For each mule wallet in the period: when was it first alerted, and at what cost.""" + df = send.assign(score=score) + mules = df[df["receiver_id"].isin(df.loc[df["y_mule"] == 1, "receiver_id"].unique())] + detected, before_any, victims_before, taka_before, ahead = 0, 0, [], 0.0, [] + missed_taka = 0.0 + for wid, rows in mules.groupby("receiver_id", sort=False): + hit = rows["score"].to_numpy() >= thr + loss = rows["y_loss"].to_numpy() + amount = rows["amount"].to_numpy() + if not hit.any(): + missed_taka += float(amount[loss == 1].sum()) + continue + first = int(np.argmax(hit)) + detected += 1 + n_before = int(loss[:first].sum()) + victims_before.append(n_before) + before_any += n_before == 0 + taka_before += float(amount[:first][loss[:first] == 1].sum()) + if wid in flagged: + ahead.append((flagged[wid] - rows["t"].iloc[first]) / 3600) + return { + "mule_wallets": int(mules["receiver_id"].nunique()), + "detected": detected, + "detected_before_any_victim_paid": before_any, + "victim_transfers_before_detection": int(sum(victims_before)), + "taka_paid_to_detected_mules_before_detection": round(taka_before), + "taka_paid_to_undetected_mules": round(missed_taka), + "median_hours_ahead_of_victim_report": round(float(np.median(ahead)), 1) if ahead else None, + } + + +def evaluate(inp: Inputs, run: dict, flagged: dict, cells: set[str], confirmed: float) -> dict: + send, matches = run["send"], run["matches"] + is_b = (send["fold"] == "val_b").to_numpy() + is_t = (send["fold"] == "test").to_numpy() + base = send["mule"].to_numpy() + out = {} + base_b = _wallet_scores(send[is_b], base[is_b]) + allowed = int(((base_b["score"] >= inp.threshold) & (base_b["y"] == 0)).sum()) + neg_b = int((base_b["y"] == 0).sum()) + detected_by_base: set[str] = set() + for arm, kinds in ALL_ARMS.items(): + c = np.array([signal([m for m in ms if m.kind in kinds]) for ms in matches]) + if not kinds: + t_mule, t_cons = inp.threshold, np.inf + elif arm in RULE_ARMS: + t_mule, t_cons = inp.threshold, confirmed + else: + t_mule, t_cons = _choose_thresholds( + send[is_b], base[is_b], c[is_b], inp.threshold, allowed + ) + score, thr = _fused(base, c, t_mule, t_cons), 1.0 + by = _wallet_scores(send[is_t], score[is_t]) + alerted = by["score"] >= thr + tp = alerted & (by["y"] == 1) + fp = alerted & (by["y"] == 0) + caught = set(by.index[tp]) + if not kinds: + detected_by_base = caught + matched = c[is_t] > 0 + out[arm] = { + "mule_threshold": round(t_mule, 4), + "consortium_threshold": None if np.isinf(t_cons) else round(t_cons, 4), + "val_b_fpr": round( + float( + ( + (_wallet_scores(send[is_b], score[is_b])["score"] >= thr) + & (base_b["y"] == 0) + ).sum() + ) + / neg_b, + 5, + ), # fmt: skip + "wallets_alerted": int(alerted.sum()), + "mule_wallets": int(by["y"].sum()), + "recall": round(float(tp.sum() / by["y"].sum()), 4), + "precision": round(float(tp.sum() / max(alerted.sum(), 1)), 4), + "false_alerts": int(fp.sum()), + "fpr": round(float(fp.sum() / (by["y"] == 0).sum()), 5), + "false_alerts_that_are_ring_wallets": int(len(set(by.index[fp]) & cells)), + "mules_found_only_with_consortium": len(caught - detected_by_base), + "transfers_with_a_partner_match": int(matched.sum()), + "timing": _timing(send[is_t], score[is_t], thr, flagged), + } + return out + + +def _summary(per_seed: list[dict]) -> dict: + """Mean, min and max over seeds of every number in the arm results.""" + out: dict = {} + for arm in per_seed[0]: + out[arm] = {} + flat = [_flatten(r[arm]) for r in per_seed] + for key in flat[0]: + vals = [f[key] for f in flat if f[key] is not None] + if not vals: + out[arm][key] = None + continue + out[arm][key] = {"mean": round(float(np.mean(vals)), 4), "min": min(vals), + "max": max(vals)} # fmt: skip + return out + + +def _flatten(d: dict, prefix: str = "") -> dict: + out = {} + for k, v in d.items(): + if isinstance(v, dict): + out.update(_flatten(v, f"{prefix}{k}.")) + else: + out[prefix + k] = v + return out + + +# ---------------------------------------------------------------------- main + + +def _oprf_timing(key: crypto.OprfKey, n: int = 200) -> dict: + """Measured cost of one blinded token on this machine, client and hub sides.""" + from .protocol import group_element + + elements = [group_element("msisdn", f"0171{i:07d}") for i in range(n)] + t0 = time.perf_counter() + blinded = [crypto.blind(e) for e in elements] + t1 = time.perf_counter() + evaluated = [key.evaluate(b) for b, _ in blinded] + t2 = time.perf_counter() + out = [crypto.unblind(e, r, key.public) for e, (_, r) in zip(evaluated, blinded, strict=True)] + t3 = time.perf_counter() + assert all(o == pow(e, key.secret, crypto.P) for o, e in zip(out, elements, strict=True)) + return { + "samples": n, + "client_ms_per_token": round((t1 - t0 + t3 - t2) / n * 1000, 2), + "hub_ms_per_token": round((t2 - t1) / n * 1000, 2), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Simulate the cross-provider consortium") + parser.add_argument("--data", type=Path, default=settings.data_dir / "full") + parser.add_argument("--quick", action="store_true", help="one seed, headline share only") + parser.add_argument("--workers", type=int, default=8) + args = parser.parse_args() + t0 = time.perf_counter() + inp = load_inputs(args.data) + conf = measured_confidences(inp) + print("model", inp.model_version, "| mule threshold", round(inp.threshold, 4), "|", conf) + w = inp.wallets.set_index("wallet_id") + flagged = {wid: ts for wid, ts, ok in zip(w.index, _epoch(w["flagged_at"].fillna( + pd.Timestamp(0))), w["flagged_at"].notna(), strict=True) if ok} # fmt: skip + cell = w["cell_id"] + cells = set(cell.index[cell >= 0]) + + hub_key = crypto.OprfKey.generate(KEY_EPOCH) + seeds = SEEDS[:1] if args.quick else SEEDS + shares = (HEADLINE_SHARE,) if args.quick else SIM_SHARES + token_cache: dict = {} + results: dict = {} + timing = _oprf_timing(hub_key) + t_tok = 0.0 + demo = None + with ProcessPoolExecutor(args.workers) as pool: + + def mapper(fn, items): + items = list(items) + return pool.map(fn, items, chunksize=max(1, len(items) // (args.workers * 4))) + + for share in shares: + per_seed = [] + for seed in seeds: + market = build_market(inp.wallets, seed, share) + # Each provider tokenises its own customers' identifiers through the + # blinded OPRF once; the cache is shared because tokens do not depend on + # who asked, and the experiment re-runs the market many times. + tt = time.perf_counter() + _bootstrap_tokens(inp, market, hub_key, token_cache, mapper) + t_tok += time.perf_counter() - tt + keep = share == HEADLINE_SHARE and seed == seeds[0] + run = run_consortium(inp, market, conf, hub_key, token_cache, mapper, keep=keep) + res = evaluate(inp, run, flagged, cells, conf["confirmed"]) + res["cells_spread"] = market.cell_spread(cell[cell >= 0]) + per_seed.append(res) + if keep: + demo = (market, run) + print(f"share {share} seed {seed}:", + {a: (r["recall"], r["fpr"], r["timing"]["detected_before_any_victim_paid"]) + for a, r in res.items() if a in ALL_ARMS}) # fmt: skip + spread = [r.pop("cells_spread") for r in per_seed] + results[f"{share:.2f}"] = { + "summary": _summary(per_seed), + "per_seed": per_seed, + "cells_spread": spread, + } + + # Poisoning: a dishonest member lists 500 innocent customers as confirmed mules. + market = build_market(inp.wallets, seeds[0], HEADLINE_SHARE) + rng = np.random.default_rng(seeds[0]) + innocent = sorted(set(w.index[(w["cell_id"] < 0) & w["first_fraud_at"].isna()])) + poison = rng.choice(innocent, size=POISON_LISTINGS, replace=False).tolist() + _bootstrap_tokens(inp, market, hub_key, token_cache, mapper, extra=poison) + attacked = evaluate( + inp, + run_consortium(inp, market, conf, hub_key, token_cache, mapper, poison=poison), + flagged, + cells, + conf["confirmed"], + ) + + clean = results[f"{HEADLINE_SHARE:.2f}"]["per_seed"][0] + report = { + "created_at": datetime.now(UTC).isoformat(timespec="seconds"), + "model_version": inp.model_version, + "mule_wallet_threshold": inp.threshold, + "providers": [{"name": n, "display": d, "share": s} for n, d, s in PROVIDERS], + "seeds": list(seeds), + "sim_shares": list(shares), + "headline_share": HEADLINE_SHARE, + "listing_confidence": conf, + "periods": {k: [iso(_dt(a)), iso(_dt(b))] for k, (a, b) in inp.folds.items()}, + "results_by_shared_sim_rate": results, + "poisoning": { + "listings": POISON_LISTINGS, + "by": "upay", + "seed": seeds[0], + "clean": {a: _pick(clean[a]) for a in ALL_ARMS}, + "attacked": {a: _pick(attacked[a]) for a in ALL_ARMS}, + }, + "cost": { + "oprf": timing, + "tokens_made": len(token_cache), + "seconds_making_tokens": round(t_tok, 1), + }, + "seconds": round(time.perf_counter() - t0, 1), + } + out = out_dir() + out.mkdir(parents=True, exist_ok=True) + (out / "report.json").write_text(json.dumps(report, indent=2)) + if demo is not None: + from .demo import save_demo + + save_demo(out, inp, demo[0], demo[1], hub_key, report) + print(json.dumps({k: report[k] for k in ("listing_confidence", "cost", "seconds")}, indent=1)) + print(json.dumps(results[f"{HEADLINE_SHARE:.2f}"]["summary"]["msisdn_and_device"], indent=1)) + + +def _pick(r: dict) -> dict: + keys = ("recall", "false_alerts", "fpr", "mule_threshold", "consortium_threshold", + "wallets_alerted") # fmt: skip + return {k: r[k] for k in keys} + + +def _bootstrap_tokens( + inp: Inputs, market: Market, key: crypto.OprfKey, cache: dict, mapper, extra=() +) -> None: + """Each provider tokenises the identifiers of its own customers (blinded).""" + hub = Hub(KEY_EPOCH, _dt(inp.folds["val_a"][0]), key=key) + wanted: dict[str, list[tuple[str, str]]] = defaultdict(list) + for wid, p in market.provider.items(): + wanted[p].append(("msisdn", market.msisdn[wid])) + wanted[p] += [("device", d) for _, d in inp.devices.get(wid, ())] + for wid in extra: + wanted["upay"].append(("msisdn", market.msisdn[wid])) + for p, pairs in wanted.items(): + m = Member(p, hub) + m._tokens = cache + hub.register(p, p, m.key.public, _dt(inp.folds["val_a"][0]), quota_per_day=10**6) + m.tokenize(pairs, _dt(inp.folds["val_a"][0]), mapper) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/decision/business.py b/backend/src/fraudlens/decision/business.py new file mode 100644 index 0000000..bdd67e5 --- /dev/null +++ b/backend/src/fraudlens/decision/business.py @@ -0,0 +1,523 @@ +"""The threshold sweep turned into a monthly profit and loss in taka. + +`insights.py` says what a threshold does on the test period: alerts, false alerts, +victims' money stopped. This module puts a price on each of those at a provider's +own volume, so the question "is it worth running, and with how many people?" has a +number for every threshold in the sweep: + +- money kept from scammers: the scam money alerted in each tier, times the share + that tier actually stops (a warning is often clicked through; a hold rarely is); +- friction: honest payments abandoned after an interruption, and the calls to the + contact centre that interruptions cause; +- review: analyst time for every held payment, with a floor for round-the-clock cover; +- what is left: net benefit a month, prevented loss per taka of operating cost, + analysts needed, honest customers interrupted per 10,000 payments. + +Every assumption is a field of `Assumptions`, with its source or the reasoning +behind it in `ASSUMPTIONS`. The model numbers come from `insights.json` only, so +nothing is re-scored. Rates per payment are taken from the back-test and multiplied +by the provider's volume; the scam money and the true alerts are rescaled from the +simulator's scam rate (about 93 basis points of payment value, far denser than real +traffic) to the assumed real one. False alerts are not rescaled: honest customers +look the same whatever the scam rate. + + uv run python -m fraudlens.decision.business +""" + +from __future__ import annotations + +import argparse +import json +import math +from dataclasses import dataclass + +from pydantic import BaseModel, ConfigDict, Field + +from ..config import settings +from ..models import registry +from .insights import INSIGHTS_FILE + +TIERS = ("warn", "step_up", "hold") + + +class Assumptions(BaseModel): + """What the business case assumes. Every field can be overridden; bounds keep it sane.""" + + model_config = ConfigDict(extra="forbid") + + monthly_payments: float = Field(20_000_000, ge=10_000, le=2_000_000_000) + avg_payment_taka: float = Field(2_800, ge=100, le=25_000) + scam_loss_bps: float = Field(2.0, ge=0.01, le=100) + stop_warn: float = Field(0.25, ge=0, le=1) + stop_step_up: float = Field(0.50, ge=0, le=1) + stop_hold: float = Field(0.90, ge=0, le=1) + abandon_rate: float = Field(0.05, ge=0, le=1) + abandon_cost_taka: float = Field(25, ge=0, le=5_000) + contacts_warn: float = Field(0.02, ge=0, le=5) + contacts_step_up: float = Field(0.10, ge=0, le=5) + contacts_hold: float = Field(0.25, ge=0, le=5) + contact_cost_taka: float = Field(50, ge=0, le=5_000) + review_minutes: float = Field(8, ge=0.5, le=240) + analyst_monthly_cost_taka: float = Field(60_000, ge=1_000, le=2_000_000) + analyst_hours_per_month: float = Field(132, ge=20, le=250) + min_analysts: int = Field(5, ge=0, le=500) + platform_monthly_cost_taka: float = Field(500_000, ge=0, le=1_000_000_000) + reimbursement_share: float = Field(0.25, ge=0, le=1) + reputation_per_taka: float = Field(0.25, ge=0, le=10) + + +@dataclass(frozen=True) +class Spec: + label: str + unit: str + low: float # the range the sensitivity table tries + high: float + source: str + + +ASSUMPTIONS: dict[str, Spec] = { + "monthly_payments": Spec( + "Send-money and cash-out payments a month", + "payments", + 10_000_000, + 40_000_000, + "Order of magnitude for a mid-sized provider (upay scale). Bangladesh Bank's monthly " + "MFS statistics put the whole industry in the hundreds of millions of transactions a " + "month; a bKash-scale provider is roughly 300 million scored payments. Replace with " + "the provider's own MIS figure.", + ), + "avg_payment_taka": Spec( + "Average payment", + "BDT", + 1_500, + 4_000, + "The simulator's scored payments average ৳2,830 (send-money ৳2,247, cash-out " + "৳3,990); rounded down. Bangladesh Bank's industry value-per-transaction is in the " + "same range.", + ), + "scam_loss_bps": Spec( + "Scam losses, share of payment value", + "basis points", + 0.5, + 8, + "Assumed. The simulator runs at 93 bp, deliberately dense so the model has cases to " + "learn from. UK push-payment scam losses are about 1-2 bp of Faster Payments value " + "(UK Finance, Pay.UK); MFS in Bangladesh is assumed more exposed. No public " + "Bangladeshi figure exists, so this is the assumption to replace first.", + ), + "stop_warn": Spec( + "Scam money stopped by a warning", + "share", + 0.10, + 0.50, + "Assumed. A coached victim often clicks through a warning; studies of payment " + "warnings report partial effect. The customer decides, so the share is low.", + ), + "stop_step_up": Spec( + "Scam money stopped by a step-up check", + "share", + 0.30, + 0.75, + "Assumed. Re-authentication defeats most account takeovers, and the 30-minute " + "cooling-off breaks the urgency a scam script relies on; a determined victim still pays.", + ), + "stop_hold": Spec( + "Scam money stopped by a hold", + "share", + 0.75, + 0.98, + "Assumed. The money stays in the wallet while an analyst calls the customer; some " + "victims insist. Also applied to mule cash-outs held after the victim's payment.", + ), + "abandon_rate": Spec( + "Honest payments abandoned after an interruption", + "share", + 0.01, + 0.15, + "Assumed. Most honest customers read the warning and continue; a step-up or a hold " + "loses some of them to another channel.", + ), + "abandon_cost_taka": Spec( + "Cost of one abandoned honest payment", + "BDT", + 10, + 60, + "Lost fee revenue and goodwill: cash-out pays about 1.85% (৳74 on ৳4,000), " + "app send-money ৳0-5; many abandoned payments are retried later.", + ), + "contacts_warn": Spec( + "Support contacts per warning", + "contacts", + 0.0, + 0.05, + "Assumed: a warning rarely leads to a call.", + ), + "contacts_step_up": Spec( + "Support contacts per step-up check", + "contacts", + 0.05, + 0.25, + "Assumed: a paused payment brings some customers to the helpline.", + ), + "contacts_hold": Spec( + "Support contacts per hold", + "contacts", + 0.10, + 0.50, + "Assumed: the analyst calls the customer anyway (counted as review time); this is " + "the extra inbound calls.", + ), + "contact_cost_taka": Spec( + "Cost of one support contact", + "BDT", + 25, + 100, + "About six minutes of a contact-centre agent at roughly ৳35,000 a month loaded, " + "plus telephony.", + ), + "review_minutes": Spec( + "Analyst minutes per held payment", + "minutes", + 4, + 20, + "The impact page's default: read the case note, call the customer, decide.", + ), + "analyst_monthly_cost_taka": Spec( + "Analyst cost a month, loaded", + "BDT", + 40_000, + 100_000, + "A fraud-operations analyst in Dhaka at about ৳45,000 a month, plus a third for " + "benefits, seat and supervision.", + ), + "analyst_hours_per_month": Spec( + "Hours of case work per analyst a month", + "hours", + 110, + 150, + "Six hours of case work in an eight-hour shift, 22 shifts a month.", + ), + "min_analysts": Spec( + "Minimum analysts for round-the-clock cover", + "people", + 3, + 10, + "Holds have a 30-minute review target at any hour: one seat staffed 24x7 takes " + "about five people.", + ), + "platform_monthly_cost_taka": Spec( + "Running the platform a month", + "BDT", + 250_000, + 1_500_000, + "Assumed: API and stream nodes, Postgres, Redis, and part of an engineer's time " + "for monitoring and retraining.", + ), + "reimbursement_share": Spec( + "Share of scam losses the provider reimburses", + "share", + 0.0, + 1.0, + "Assumed. Bangladesh has no mandatory reimbursement rule; the UK has required 100% " + "(up to £85,000) since October 2024, the upper end of the range.", + ), + "reputation_per_taka": Spec( + "Reputational cost per taka a customer loses", + "BDT per BDT", + 0.0, + 1.0, + "Assumed: victims and the people they tell use the wallet less, and the regulator " + "pays attention. The hardest number here to defend; shown separately.", + ), +} + +_RATE_KEYS = ("alert_rate", "precision", "taka_recall", "taka_recall_with_exit_holds") + + +def _cumulative(point: dict) -> dict[str, float]: + """Per-payment rates of everything at or above one threshold.""" + rate, precision = float(point["alert_rate"]), float(point["precision"]) + return { + "true": rate * precision, + "false": rate * (1 - precision), + "recall": float(point["taka_recall"]), + "recall_exit": float(point["taka_recall_with_exit_holds"]), + } + + +def _loss_per_payment(a: Assumptions) -> float: + return a.avg_payment_taka * a.scam_loss_bps / 10_000 + + +def _synthetic_loss_per_payment(insights: dict) -> float: + return insights["at_thresholds"]["warn"]["taka_at_risk"] / insights["rows"] + + +def _scale(insights: dict, a: Assumptions) -> float: + """Real scam money per payment over the simulator's: rescales the true alerts.""" + synthetic = _synthetic_loss_per_payment(insights) + return _loss_per_payment(a) / synthetic if synthetic else 0.0 + + +def nearest(points: list[dict], threshold: float) -> int: + """The sweep point closest to `threshold`, on a log scale (as the impact page does).""" + return min(range(len(points)), key=lambda i: abs(math.log(points[i]["threshold"] / threshold))) + + +def _point(insights: dict, point: dict, a: Assumptions, scale: float) -> dict: + """The monthly money of alerting at `point` as the warn threshold. + + Step-up and hold keep the policy's thresholds, or follow the slider once it passes + them. Each tier's band is the difference between two cumulative sweep rows. + """ + t = float(point["threshold"]) + here = _cumulative(point) + at = insights["at_thresholds"] + limits = insights["thresholds"] + s = here if t >= limits["step_up"] else _cumulative(at["step_up"]) + h = here if t >= limits["hold"] else _cumulative(at["hold"]) + cum = {"warn": here, "step_up": s, "hold": h} + following = {"warn": s, "step_up": h, "hold": None} + bands = {} + for tier in TIERS: + upper, lower = cum[tier], following[tier] + bands[tier] = {key: max(upper[key] - (lower[key] if lower else 0.0), 0.0) for key in upper} + + volume = a.monthly_payments + at_risk = volume * a.avg_payment_taka * a.scam_loss_bps / 10_000 + stop = {"warn": a.stop_warn, "step_up": a.stop_step_up, "hold": a.stop_hold} + contacts = {"warn": a.contacts_warn, "step_up": a.contacts_step_up, "hold": a.contacts_hold} + + prevented = {tier: at_risk * bands[tier]["recall"] * stop[tier] for tier in TIERS} + # Only a hold stops a mule's cash-out: a warning shown to a mule changes nothing. + exit_share = max(h["recall_exit"] - h["recall"], 0.0) + prevented["mule_cash_out_held"] = at_risk * exit_share * a.stop_hold + prevented_total = sum(prevented.values()) + + interrupted = { + tier: volume * (bands[tier]["true"] * scale + bands[tier]["false"]) for tier in TIERS + } + honest = {tier: volume * bands[tier]["false"] for tier in TIERS} + abandoned = sum(honest.values()) * a.abandon_rate + support_contacts = sum(interrupted[tier] * contacts[tier] for tier in TIERS) + friction = { + "abandoned_payments": abandoned * a.abandon_cost_taka, + "support_contacts": support_contacts * a.contact_cost_taka, + } + + held = interrupted["hold"] + review_hours = held * a.review_minutes / 60 + workload_fte = review_hours / a.analyst_hours_per_month + analysts = max(math.ceil(workload_fte - 1e-9), a.min_analysts) + analyst_cost = analysts * a.analyst_monthly_cost_taka + + operating = sum(friction.values()) + analyst_cost + a.platform_monthly_cost_taka + missed = at_risk - prevented_total + return { + "threshold": t, + "alert_rate": float(point["alert_rate"]), + "interrupted": {tier: round(n) for tier, n in interrupted.items()}, + "interrupted_total": round(sum(interrupted.values())), + "honest_interrupted": round(sum(honest.values())), + "honest_per_10k": round(sum(bands[t]["false"] for t in TIERS) * 10_000, 2), + "held_for_review": round(held), + "prevented": {k: round(v) for k, v in prevented.items()}, + "prevented_total": round(prevented_total), + "prevented_share": round(prevented_total / at_risk, 4) if at_risk else 0.0, + "missed": round(missed), + "friction": {k: round(v) for k, v in friction.items()}, + "friction_total": round(sum(friction.values())), + "abandoned_payments": round(abandoned), + "support_contacts": round(support_contacts), + "review_hours": round(review_hours), + "workload_fte": round(workload_fte, 1), + "analysts": int(analysts), + "analyst_cost": round(analyst_cost), + "platform_cost": round(a.platform_monthly_cost_taka), + "operating_cost": round(operating), + # Customers and provider together: every taka a customer keeps counts, plus the + # reputational cost avoided. + "net_benefit": round(prevented_total * (1 + a.reputation_per_taka) - operating), + # The provider's own cash: only what it would have reimbursed, plus reputation. + "provider_net_benefit": round( + prevented_total * (a.reimbursement_share + a.reputation_per_taka) - operating + ), + "prevented_per_taka_cost": round(prevented_total / operating, 2) if operating else None, + "missed_cost_provider": round(missed * (a.reimbursement_share + a.reputation_per_taka)), + } + + +def business_case(insights: dict, assumptions: Assumptions | None = None) -> dict: + """The monthly money at every threshold of the sweep, under `assumptions`.""" + a = assumptions or Assumptions() + sweep = insights.get("impact") or [] + if not sweep: + raise ValueError("the insights report has no threshold sweep") + scale = _scale(insights, a) + points = [_point(insights, p, a, scale) for p in sweep] + today = nearest(sweep, insights["thresholds"]["warn"]) + best = max(range(len(points)), key=lambda i: points[i]["net_benefit"]) + return { + "model_version": insights.get("model_version"), + "policy_version": insights.get("policy_version"), + "rows": insights["rows"], + "days": insights["days"], + "assumptions": a.model_dump(), + "defaults": Assumptions().model_dump(), + "sources": [ + { + "key": key, + "label": s.label, + "unit": s.unit, + "low": s.low, + "high": s.high, + "source": s.source, + } + for key, s in ASSUMPTIONS.items() + ], + "synthetic": { + "scam_loss_per_payment": round(_synthetic_loss_per_payment(insights), 2), + "scale": round(scale, 5), + }, + "scam_loss_at_risk": round(a.monthly_payments * _loss_per_payment(a)), + "today_index": today, + "best_index": best, + # The thresholds in force, measured exactly (the sweep only comes near them). + "policy": _point(insights, _policy_row(insights), a, scale), + "points": points, + "sensitivity": sensitivity(insights, a), + "break_even_bps": { + "net_benefit": break_even_bps(insights, a, "net_benefit"), + "provider_net_benefit": break_even_bps(insights, a, "provider_net_benefit"), + }, + } + + +def break_even_bps(insights: dict, a: Assumptions, key: str = "net_benefit") -> float | None: + """The scam rate (basis points of payment value) below which running the policy in + force costs more than it saves, everything else held. None if it never pays off + within the field's range.""" + row = _policy_row(insights) + low, high = 0.01, 100.0 # the field's bounds + + def net(bps: float) -> float: + changed = a.model_copy(update={"scam_loss_bps": bps}) + return _point(insights, row, changed, _scale(insights, changed))[key] + + if net(high) < 0: + return None + if net(low) >= 0: + return low + for _ in range(50): + mid = (low + high) / 2 + low, high = (mid, high) if net(mid) < 0 else (low, mid) + return round(high, 3) + + +def _policy_row(insights: dict) -> dict: + """The warn threshold in force, as a sweep row: exact, not the nearest sweep point.""" + return {**insights["at_thresholds"]["warn"], "threshold": insights["thresholds"]["warn"]} + + +def sensitivity(insights: dict, assumptions: Assumptions, row: dict | None = None) -> dict: + """Tornado table at one threshold (default: the policy's warn threshold). + + Each assumption is set to its low and then its high value with the others held; + rows are sorted by how far that swings the net benefit, largest first. The + provider's own net benefit is given too, since reimbursement only moves that one. + """ + row = row or _policy_row(insights) + base = _point(insights, row, assumptions, _scale(insights, assumptions)) + rows = [] + for key, spec in ASSUMPTIONS.items(): + out = {} + for side, value in (("low", spec.low), ("high", spec.high)): + changed = assumptions.model_copy(update={key: type(getattr(assumptions, key))(value)}) + point = _point(insights, row, changed, _scale(insights, changed)) + out[f"net_{side}"] = point["net_benefit"] + out[f"provider_{side}"] = point["provider_net_benefit"] + rows.append( + { + "key": key, + "label": spec.label, + "unit": spec.unit, + "low": spec.low, + "high": spec.high, + **out, + "swing": abs(out["net_high"] - out["net_low"]), + "provider_swing": abs(out["provider_high"] - out["provider_low"]), + } + ) + rows.sort(key=lambda r: (r["swing"], r["provider_swing"]), reverse=True) + return { + "threshold": float(row["threshold"]), + "net_benefit": base["net_benefit"], + "provider_net_benefit": base["provider_net_benefit"], + "rows": rows, + } + + +def _row(p: dict, mark: str = "") -> str: + return ( + f"{p['threshold']:>10.5f} {p['alert_rate']:>8.4%} {p['prevented_total']:>13,} " + f"{p['friction_total']:>11,} {p['analysts']:>8} {p['operating_cost']:>12,} " + f"{p['net_benefit']:>13,} {p['provider_net_benefit']:>13,} " + f"{p['prevented_per_taka_cost'] or 0:>7.2f} {p['honest_per_10k']:>10.2f}{mark}" + ) + + +def print_summary(case: dict) -> None: + print( + f"model {case['model_version']}, policy {case['policy_version']}: " + f"{case['assumptions']['monthly_payments']:,.0f} payments a month, " + f"scam money at risk ৳{case['scam_loss_at_risk']:,} a month " + f"(synthetic scale {case['synthetic']['scale']})" + ) + print( + f"{'threshold':>10} {'alerted':>8} {'prevented':>13} {'friction':>11} {'analysts':>8} " + f"{'operating':>12} {'net':>13} {'provider net':>13} {'per ৳1':>7} {'honest/10k':>10}" + ) + for i, p in enumerate(case["points"]): + mark = " <- nearest to today" if i == case["today_index"] else "" + print(_row(p, mark + (" <- best" if i == case["best_index"] else ""))) + policy = case["policy"] + print(_row(policy, " <- policy in force (exact)")) + print(f"\nat the policy in force: prevented {json.dumps(policy['prevented'])}") + print( + f" friction {json.dumps(policy['friction'])}, held {policy['held_for_review']:,}, " + f"review hours {policy['review_hours']:,}, workload {policy['workload_fte']} FTE, " + f"interrupted {json.dumps(policy['interrupted'])}" + ) + sens = case["sensitivity"] + print( + f"\nsensitivity at the policy's warn threshold: net ৳{sens['net_benefit']:,}, " + f"provider ৳{sens['provider_net_benefit']:,}" + ) + for row in sens["rows"]: + print( + f" {row['label']:<48} {row['low']:>12,} -> ৳{row['net_low']:>12,} " + f"(provider ৳{row['provider_low']:>12,}) {row['high']:>12,} -> " + f"৳{row['net_high']:>12,} (provider ৳{row['provider_high']:>12,})" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="The threshold sweep as a monthly P&L in taka") + parser.add_argument("--model", default=None, help="model version (default: current)") + parser.add_argument( + "--set", action="append", default=[], metavar="KEY=VALUE", help="override an assumption" + ) + parser.add_argument("--json", action="store_true", help="print the whole result as JSON") + args = parser.parse_args() + version = args.model or registry.current_version(settings.models_dir) + insights = json.loads((settings.models_dir / version / INSIGHTS_FILE).read_text()) + overrides = dict(item.split("=", 1) for item in args.set) + case = business_case(insights, Assumptions(**overrides)) + if args.json: + print(json.dumps(case, indent=2)) + else: + print_summary(case) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/decision/customer.py b/backend/src/fraudlens/decision/customer.py new file mode 100644 index 0000000..0e6d4f7 --- /dev/null +++ b/backend/src/fraudlens/decision/customer.py @@ -0,0 +1,53 @@ +"""Which scam pattern a customer-facing screen should name. + +The policy's customer message is a fixed text per scenario and tier. A good +warning also asks the one question that fits the scam in front of the customer +("did someone claiming to be from your bank ask you to send this?"). This picks +that question's key from what the decision already recorded: the rules that +fired, its scenario, and the typology of the past cases it most resembles. + +Nothing is scored here, and the key says no more than the message already does: +no score, no reason detail, nothing about the receiving wallet beyond what the +recipient check would say. +""" + +from __future__ import annotations + +from collections import defaultdict + +CUES = ( + "reported_recipient", # the receiving wallet is confirmed fraud (rule R01) + "impersonation", # someone posing as upay, bKash, a bank or an official + "prize", # a prize or lottery that needs a fee + "investment", # an investment that promises high returns + "wrong_send", # "I sent you money by mistake, send it back" + "not_you", # account takeover, or use from an unusual place or network + "generic", +) + +_TYPOLOGY = { + "impersonation": "impersonation", + "lottery_fee": "prize", + "investment_scam": "investment", + "wrong_send": "wrong_send", + "account_takeover": "not_you", +} + + +def cue(tier: str, scenario: str | None, detail: dict | None) -> str | None: + """The pattern key for an alert, or None for a payment that went straight through.""" + if tier == "allow": + return None + detail = detail or {} + fired = {r.get("id") for r in detail.get("rule_trace") or [] if r.get("status") == "fired"} + if "R01_RECIPIENT_CONFIRMED_FRAUD" in fired: + return "reported_recipient" + if scenario in ("takeover", "unusual_access"): + return "not_you" + weight: dict[str, float] = defaultdict(float) + for case in detail.get("similar_cases") or []: + if case.get("typology"): + weight[case["typology"]] += case.get("similarity") or 0.0 + if weight: + return _TYPOLOGY.get(max(weight, key=weight.__getitem__), "generic") + return "generic" diff --git a/backend/src/fraudlens/decision/engine.py b/backend/src/fraudlens/decision/engine.py index d58e752..8fe3682 100644 --- a/backend/src/fraudlens/decision/engine.py +++ b/backend/src/fraudlens/decision/engine.py @@ -80,6 +80,9 @@ class Decision: similar_cases: tuple[dict, ...] evidence: dict narrative: str | None + # The policy segment whose cut-offs set the model's tier, with those cut-offs; None + # when the policy's own cut-offs applied (or in rules-only mode). + segment: dict | None = None def to_dict(self) -> dict: return asdict(self) @@ -144,7 +147,8 @@ def decide( else [] ) - risk_score = display_score(risk, self.thresholds) if risk is not None else None + # Anchored to the cut-offs that applied, so 80 still means "held" inside a segment. + risk_score = display_score(risk, outcome.thresholds) if risk is not None else None band = _BANDS[outcome.model_tier] if outcome.model_tier else "not scored" evidence = { "transaction": { @@ -201,6 +205,13 @@ def decide( similar_cases=tuple(similar), evidence=evidence, narrative=template(evidence) if alert else None, + segment={ + "id": outcome.segment.id, + "description": outcome.segment.description, + "thresholds": dict(outcome.thresholds), + } + if outcome.segment + else None, ) # ------------------------------------------------------------- internals diff --git a/backend/src/fraudlens/decision/mitigation.py b/backend/src/fraudlens/decision/mitigation.py new file mode 100644 index 0000000..84ebcb1 --- /dev/null +++ b/backend/src/fraudlens/decision/mitigation.py @@ -0,0 +1,644 @@ +"""Segment thresholds for young wallets: fitted with a stated objective, measured on test. + +The fairness report (insights.py) shows that honest payments involving a wallet under +30 days old are interrupted far more often than others. A policy can give such +payments their own cut-offs (`segments:` in the policy file). This module chooses +those cut-offs and measures what they cost. + +- Fitted on the validation folds (val_a + val_b), never on test. +- Objective: make the larger of the two young-wallet false-alert ratios (sender under + 30 days, receiving wallet under 30 days, each against honest payments that involve + no young wallet, a rate the segments cannot move) as small as possible, subject to + losing at most MAX_RECALL_LOSS of victim transfers alerted, the same of victim + money alerted, and MAX_HOLD_RECALL_LOSS of victim transfers held. Ratios within + GAP_TOLERANCE count as equal; ties go to the smaller sum of the two ratios, then + the smaller false-hold ratio, then the smaller change. The report also gives the + ratios against the overall rate, as the fairness report does. +- Only candidates the policy guard admits can be chosen: a segment's warn cut-off may + not exceed the model's hold cut-off. Inadmissible candidates are still measured, so + the frontier shows what closing the gap further would cost. +- Evaluated on the test period by running the base policy and the target policy + through the real decision code, with 95% intervals from a bootstrap that resamples + sending customers (all of a customer's payments together). + + uv run python -m fraudlens.decision.mitigation # fit, then evaluate policy v3 +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import math +from collections import Counter +from pathlib import Path + +import numpy as np +import pandas as pd + +from ..config import settings +from ..models import registry +from ..models.data import load_frame +from ..models.metrics import alert_outcomes +from .evaluate import add_context, decide_batch +from .insights import _group_table +from .policy import _OPS, POLICY_DIR, RANK, Condition, Policy, Segment, load_policy + +MITIGATION_FILE = "fairness_mitigation.json" +TARGET_POLICY = "v3" +FIT_FOLDS = ("val_a", "val_b") +YOUNG_DAYS = 30.0 +# Warn cut-off as a multiple of the model's. The largest admissible one is the model's +# hold / warn ratio (25.1 for model v4); the larger ones only draw the frontier. +WARN_SCALES = (1, 1.5, 2, 3, 4, 6, 8, 12, 16, 20, 25, 40, 60, 100, 150, 250) +HOLD_SCALES = (1, 1.5, 2, 3, 4) +MAX_RECALL_LOSS = 0.005 # 0.5 percentage points +MAX_HOLD_RECALL_LOSS = 0.01 # 1 percentage point +GAP_TOLERANCE = 0.1 +BOOTSTRAP = 1000 +SEED = 7 + +# --------------------------------------------------------------------- measures + + +def _columns(rows: pd.DataFrame, rank: np.ndarray) -> dict[str, np.ndarray]: + """Per-row indicators whose sums give every rate reported here.""" + legit = rows["y"].to_numpy() == 0 + victim = rows["y_loss"].to_numpy() == 1 + send = (rows["type"] == "SEND_MONEY").to_numpy() + young_sender = rows["s_age_days"].to_numpy(dtype=np.float64) < YOUNG_DAYS + young_receiver = send & (rows["r_age_days"].to_numpy(dtype=np.float64) < YOUNG_DAYS) + taka = np.where(victim, rows["amount"].to_numpy(dtype=np.float64), 0.0) + alert, hold = rank > 0, rank == RANK["hold"] + groups = { + "": legit, + "_send": legit & send, + "_ys": legit & young_sender, + "_yr": legit & young_receiver, + "_ref": legit & ~young_sender & ~young_receiver, # no young wallet: no segment applies + } + cols = {"rows": np.ones(len(rows)), "alert": alert, "hold": hold} + for suffix, mask in groups.items(): + cols["legit" + suffix] = mask + cols["fa" + suffix] = mask & alert + cols["fh" + suffix] = mask & hold + cols |= {"victim": victim, "va": victim & alert, "vh": victim & hold} + cols |= {"taka": taka, "ta": taka * alert, "th": taka * hold} + return {name: np.asarray(values, dtype=np.float64) for name, values in cols.items()} + + +def _metrics(s: dict) -> dict: + """Rates from summed indicators. Works on numbers or on arrays of them.""" + + def div(a, b): + with np.errstate(divide="ignore", invalid="ignore"): + return np.divide(a, b) + + far, far_send = div(s["fa"], s["legit"]), div(s["fa_send"], s["legit_send"]) + fhr, fhr_send = div(s["fh"], s["legit"]), div(s["fh_send"], s["legit_send"]) + out = { + "false_alert_rate": far, + "false_hold_rate": fhr, + "sender_young_false_alert_rate": div(s["fa_ys"], s["legit_ys"]), + "sender_young_false_hold_rate": div(s["fh_ys"], s["legit_ys"]), + "receiver_young_false_alert_rate": div(s["fa_yr"], s["legit_yr"]), + "receiver_young_false_hold_rate": div(s["fh_yr"], s["legit_yr"]), + "victim_recall": div(s["va"], s["victim"]), + "victim_recall_hold": div(s["vh"], s["victim"]), + "taka_recall": div(s["ta"], s["taka"]), + "taka_recall_hold": div(s["th"], s["taka"]), + "alert_rate": div(s["alert"], s["rows"]), + "hold_rate": div(s["hold"], s["rows"]), + } + # Same definitions as the fairness report: the sender against everyone, the + # receiving wallet against all transfers. + out["sender_young_ratio"] = div(out["sender_young_false_alert_rate"], far) + out["receiver_young_ratio"] = div(out["receiver_young_false_alert_rate"], far_send) + out["largest_young_ratio"] = np.maximum(out["sender_young_ratio"], out["receiver_young_ratio"]) + # The fit's yardstick: against payments with no young wallet, whose rate the + # segments cannot move. (Against the overall rate, interrupting young wallets less + # lowers the overall rate too, so a fix for one group raises the other's ratio.) + ref = div(s["fa_ref"], s["legit_ref"]) + out["reference_false_alert_rate"] = ref + out["sender_young_ratio_ref"] = div(out["sender_young_false_alert_rate"], ref) + out["receiver_young_ratio_ref"] = div(out["receiver_young_false_alert_rate"], ref) + out["largest_young_ratio_ref"] = np.maximum( + out["sender_young_ratio_ref"], out["receiver_young_ratio_ref"] + ) + out["summed_young_ratio_ref"] = out["sender_young_ratio_ref"] + out["receiver_young_ratio_ref"] + out["sender_young_hold_ratio"] = div(out["sender_young_false_hold_rate"], fhr) + out["receiver_young_hold_ratio"] = div(out["receiver_young_false_hold_rate"], fhr_send) + out["largest_young_hold_ratio"] = np.maximum( + out["sender_young_hold_ratio"], out["receiver_young_hold_ratio"] + ) + return out + + +def _sums(cols: dict[str, np.ndarray], mask: np.ndarray | None = None) -> dict[str, float]: + return {k: float(v.sum() if mask is None else v[mask].sum()) for k, v in cols.items()} + + +# ------------------------------------------------------------------- the grid + + +def _mask(conditions: tuple[Condition, ...], rows: pd.DataFrame) -> np.ndarray: + """Vectorised `evaluate(...) == 'fired'`: a missing value never matches.""" + hit = np.ones(len(rows), dtype=bool) + for c in conditions: + values = rows[c.field].to_numpy(dtype=np.float64) + hit &= ~np.isnan(values) & _OPS[c.op](values, c.value) + return hit + + +def segment_masks(segments: tuple[Segment, ...], rows: pd.DataFrame) -> list[np.ndarray]: + """Which rows each segment decides; the first segment that matches wins.""" + taken = np.zeros(len(rows), dtype=bool) + masks = [] + for segment in segments: + mask = rows["type"].isin(segment.applies_to).to_numpy() & _mask(segment.when, rows) & ~taken + masks.append(mask) + taken |= mask + return masks + + +def scales(base: dict[str, float], warn: float, hold: float) -> dict[str, float]: + """Full scale for a segment: step-up sits between the new warn and hold cut-offs.""" + step_up = min(max(base["step_up"], base["warn"] * warn), base["hold"] * hold) + # Four decimals, so the policy file can carry the exact number; rounded towards + # the middle so the cut-offs stay in order. + exact = step_up / base["step_up"] + rounded = math.ceil(exact * 1e4) / 1e4 + if base["step_up"] * rounded > base["hold"] * hold: + rounded = math.floor(exact * 1e4) / 1e4 + return {"warn": float(warn), "step_up": rounded, "hold": float(hold)} + + +def admissible(base: dict[str, float], scale: dict[str, float]) -> bool: + """The policy guard (Segment.thresholds), without raising.""" + t = {tier: base[tier] * scale[tier] for tier in scale} + return 0 < t["warn"] <= t["step_up"] <= t["hold"] < 1 and t["warn"] <= base["hold"] + + +def segment_thresholds(base: dict[str, float], scale: dict[str, float]) -> dict[str, float]: + """The cut-offs the engine would use; beyond the guard, the plain scaled ones, to + show what removing the guard would do.""" + if admissible(base, scale): + return Segment.model_construct(id="FIT", scale=scale).thresholds(base) + return {tier: base[tier] * value for tier, value in scale.items()} + + +def model_rank(risk: np.ndarray, thresholds: dict[str, float]) -> np.ndarray: + return np.select( + [risk >= thresholds["hold"], risk >= thresholds["step_up"], risk >= thresholds["warn"]], + [RANK["hold"], RANK["step_up"], RANK["warn"]], + RANK["allow"], + ) + + +def rule_floor(policy: Policy, outcomes: list) -> np.ndarray: + """The tier the rules lift each row to; with no cap rule, final = max(model, floor).""" + if any(rule.effect == "cap_at" for rule in policy.rules): + raise NotImplementedError("fitting assumes the policy has no cap rule") + return np.array( + [ + max((RANK[r.tier] for r in o.fired if r.effect == "raise_to"), default=0) + for o in outcomes + ] + ) + + +def grid( + rows: pd.DataFrame, + risk: np.ndarray, + floor: np.ndarray, + base: dict[str, float], + masks: list[np.ndarray], +) -> tuple[list[tuple[float, float]], dict[str, np.ndarray]]: + """Every combination of (warn, hold) scale per segment, summed indicators per combination. + + Segments are disjoint, so a combination's sums are those of the rows no segment + takes plus each segment's sums under its own scale. + """ + options = list(itertools.product(WARN_SCALES, HOLD_SCALES)) + rest = ~np.any(masks, axis=0) + total = _sums(_columns(rows[rest], np.maximum(model_rank(risk[rest], base), floor[rest]))) + per_segment = [] + for mask in masks: + seg_rows, seg_risk, seg_floor = rows[mask], risk[mask], floor[mask] + sums = [] + for warn, hold in options: + t = segment_thresholds(base, scales(base, warn, hold)) + rank = np.maximum(model_rank(seg_risk, t), seg_floor) + sums.append(_sums(_columns(seg_rows, rank))) + per_segment.append(sums) + # One axis per segment: sums[name] has shape (len(options),) * n_segments. + combined = {} + for name in total: + acc = np.asarray(total[name]) + for i, sums in enumerate(per_segment): + shape = [1] * len(masks) + shape[i] = len(options) + acc = acc + np.array([s[name] for s in sums]).reshape(shape) + combined[name] = acc + return options, combined + + +def _pareto(gap: np.ndarray, recall: np.ndarray) -> np.ndarray: + """Points no other point beats on both: smaller gap and larger recall.""" + order = np.lexsort((-recall, gap)) + keep, best = np.zeros(len(gap), dtype=bool), -np.inf + for i in order: + if recall[i] > best + 1e-12: + keep[i], best = True, recall[i] + return keep + + +def fit( + fit_metrics: dict, + test_metrics: dict, + options: list[tuple[float, float]], + base: dict[str, float], + segment_ids: list[str], +) -> dict: + """Choose one (warn, hold) scale per segment; also the frontiers it was chosen from.""" + shape = fit_metrics["largest_young_ratio"].shape + index = list(np.ndindex(*shape)) + ok = np.array([all(admissible(base, scales(base, *options[i])) for i in ix) for ix in index]) + m = {k: np.asarray(v).reshape(-1) for k, v in fit_metrics.items()} + t = {k: np.asarray(v).reshape(-1) for k, v in test_metrics.items()} + # The base policy is the combination with every scale at 1. + origin = index.index(tuple(options.index((1, 1)) for _ in segment_ids)) + + def kept(key: str, loss: float) -> np.ndarray: + # A rate with nothing to measure (no victims in the rows) cannot be lost. + with np.errstate(invalid="ignore"): + return ~(m[key] < m[key][origin] - loss - 1e-12) + + meets = ( + ok + & kept("victim_recall", MAX_RECALL_LOSS) + & kept("taka_recall", MAX_RECALL_LOSS) + & kept("victim_recall_hold", MAX_HOLD_RECALL_LOSS) + ) + change = np.array([sum(options[i][0] + options[i][1] for i in ix) for ix in index]) + candidates = np.flatnonzero(meets) # never empty: the base policy meets every constraint + gap = np.nan_to_num(m["largest_young_ratio_ref"], nan=np.inf) + total = np.nan_to_num(m["summed_young_ratio_ref"], nan=np.inf) + near = candidates[gap[candidates] <= gap[candidates].min() + GAP_TOLERANCE] + near = near[total[near] <= total[near].min() + GAP_TOLERANCE] + chosen = min(near, key=lambda i: (round(float(m["largest_young_hold_ratio"][i]), 2), change[i])) + + def point(i: int) -> dict: + return { + "scales": { + sid: dict(zip(("warn", "hold"), options[ix], strict=True)) + for sid, ix in zip(segment_ids, index[i], strict=True) + }, + "admissible": bool(ok[i]), + "meets_constraints": bool(meets[i]), + "chosen": bool(i == chosen), + "fit": {k: _r(m[k][i]) for k in _FRONTIER_KEYS}, + "test": {k: _r(t[k][i]) for k in _FRONTIER_KEYS}, + } + + # Alert frontier: warn scales vary, hold scales as chosen. Hold frontier: the reverse. + def path(vary: int) -> list[dict]: + """Every segment given the same warn (vary=0) or hold (vary=1) scale, in turn, with + the other scale as chosen: one line from the base policy to beyond the guard.""" + fixed = index[chosen] + picked = [] + for value in sorted({option[vary] for option in options}): + want = tuple( + options.index((value, options[f][1]) if vary == 0 else (options[f][0], value)) + for f in fixed + ) + picked.append(index.index(want)) + return [point(i) for i in picked] + + return { + "candidates": len(index), + "admissible": int(ok.sum()), + "meeting_constraints": int(meets.sum()), + "base": point(origin), + "chosen": point(chosen), + # Pareto-best points on validation, by gap and victim recall; then two paths. + "frontier": [ + point(i) + for i in sorted( + np.flatnonzero(_pareto(m["largest_young_ratio_ref"], m["victim_recall"])), + key=lambda i: m["largest_young_ratio_ref"][i], + ) + ], + "path_warn": path(0), + "path_hold": path(1), + } + + +_FRONTIER_KEYS = ( + "largest_young_ratio_ref", + "sender_young_ratio_ref", + "receiver_young_ratio_ref", + "largest_young_ratio", + "sender_young_ratio", + "receiver_young_ratio", + "largest_young_hold_ratio", + "receiver_young_false_hold_rate", + "false_alert_rate", + "victim_recall", + "victim_recall_hold", + "taka_recall", + "taka_recall_hold", +) + + +def _r(value, digits: int = 5): + value = float(value) + return None if np.isnan(value) else round(value, digits) + + +# ------------------------------------------------------------------ evaluation + + +_REPORTED = ( + ("sender_young_false_alert_rate", "Honest payments from senders under 30 days, interrupted"), + ("sender_young_ratio", "… times the overall rate"), + ("receiver_young_false_alert_rate", "Honest payments to wallets under 30 days, interrupted"), + ("receiver_young_ratio", "… times the overall rate"), + ("sender_young_ratio_ref", "Young senders: times the rate with no young wallet"), + ("receiver_young_ratio_ref", "Young receivers: times the rate with no young wallet"), + ("receiver_young_false_hold_rate", "Honest payments to wallets under 30 days, held"), + ("sender_young_false_hold_rate", "Honest payments from senders under 30 days, held"), + ("false_alert_rate", "All honest payments, interrupted"), + ("false_hold_rate", "All honest payments, held"), + ("victim_recall", "Victim transfers alerted (warn or above)"), + ("victim_recall_hold", "Victim transfers held"), + ("taka_recall", "Victim money alerted"), + ("taka_recall_hold", "Victim money held"), + ("alert_rate", "All payments alerted"), + ("hold_rate", "All payments held"), +) + + +def bootstrap( + rows: pd.DataFrame, before: np.ndarray, after: np.ndarray, reps: int = BOOTSTRAP +) -> dict: + """Point values and 95% intervals, before, after and the difference. + + Resamples sending customers with Poisson(1) weights, so that a customer's + payments stay together; resampling single payments would understate the spread. + """ + a, b = _columns(rows, before), _columns(rows, after) + names = list(a) + x = np.column_stack([a[n] for n in names] + [b[n] for n in names]) + codes, uniques = pd.factorize(rows["sender_id"]) + rng = np.random.default_rng(SEED) + draws = [] + for start in range(0, reps, 100): + n = min(100, reps - start) + w = rng.poisson(1.0, size=(n, len(uniques))).astype(np.float64)[:, codes] + draws.append(w @ x) + sums = np.vstack(draws) + k = len(names) + mb = _metrics({n: sums[:, i] for i, n in enumerate(names)}) + ma = _metrics({n: sums[:, k + i] for i, n in enumerate(names)}) + pb, pa = _metrics(_sums(a)), _metrics(_sums(b)) + + def ci(values): + if np.isnan(values).all(): # nothing to measure in these rows + return [None, None] + lo, hi = np.nanpercentile(values, [2.5, 97.5]) + return [_r(lo), _r(hi)] + + with np.errstate(invalid="ignore"): + return _summary(reps, pb, pa, mb, ma, ci) + + +def _summary(reps, pb, pa, mb, ma, ci) -> dict: + return { + "reps": reps, + "resampled": "sending customers", + "metrics": [ + { + "key": key, + "label": label, + "before": _r(pb[key]), + "after": _r(pa[key]), + "difference": _r(pa[key] - pb[key]), + "before_ci": ci(mb[key]), + "after_ci": ci(ma[key]), + "difference_ci": ci(ma[key] - mb[key]), + } + for key, label in _REPORTED + ], + } + + +def intersectional( + rows: pd.DataFrame, before: np.ndarray, after: np.ndarray, wallets: pd.DataFrame +) -> dict: + """Account age x channel x area, for the sender and the receiving wallet, both policies. + + Same table as the fairness report (rates against the overall rate of the same + rows; groups under MIN_GROUP_ROWS honest payments marked too small). + """ + attributes = wallets.set_index("wallet_id")[["channel", "area_type"]] + sends = (rows["type"] == "SEND_MONEY").to_numpy() + out = {} + for side, part, party, age in ( + ("sender", np.ones(len(rows), dtype=bool), "sender_id", "s_age_days"), + ("receiver", sends, "receiver_id", "r_age_days"), + ): + sub = rows[part].reset_index(drop=True) + young = np.where( + sub[age].to_numpy(dtype=np.float64) < YOUNG_DAYS, "under 30 days", "30 days+" + ) + labels = ( + pd.Series(young) + + " · " + + sub[party].map(attributes["channel"]).fillna("unknown").str.upper().to_numpy() + + " · " + + sub[party].map(attributes["area_type"]).fillna("unknown").to_numpy() + ).to_numpy(str) + b = _group_table(labels, sub, before[part]) + a = {row["group"]: row for row in _group_table(labels, sub, after[part])} + out[side] = [ + { + **row, + "false_alert_rate_after": a[row["group"]]["false_alert_rate"], + "false_hold_rate_after": a[row["group"]]["false_hold_rate"], + "ratio_to_overall_after": a[row["group"]]["ratio_to_overall"], + "victim_transfers_alerted_after": a[row["group"]]["victim_transfers_alerted"], + } + for row in b + ] + return out + + +def _outcomes(rows: pd.DataFrame, rank: np.ndarray) -> dict: + keys = ( + "alerts_per_day", + "precision", + "case_recall", + "loss_txn_recall", + "taka_recall", + "taka_stopped", + ) + return { + tier: {k: v for k, v in alert_outcomes(rows, rank >= RANK[tier]).items() if k in keys} + for tier in ("warn", "hold") + } + + +def run( + data_dir: Path, + policy_version: str = TARGET_POLICY, + model: str | None = None, + models_root: Path | None = None, + reps: int = BOOTSTRAP, +) -> dict: + target = load_policy(policy_version) + if not target.segments: + raise ValueError(f"policy {policy_version} has no segments to fit or evaluate") + base_policy = target.model_copy(update={"segments": ()}) + bundle = registry.load(model, models_root) + model_dir = (models_root or registry.models_dir()) / bundle.version + base = base_policy.resolve_thresholds(bundle.manifest) + + frame = load_frame(data_dir) + frame = frame[~frame["ambiguous"]].reset_index(drop=True) + frame = add_context(frame, pd.read_parquet(data_dir / "wallet_flags.parquet")) + ids = [s.id for s in target.segments] + + measured = {} + for name, folds in (("fit", FIT_FOLDS), ("test", ("test",))): + rows = frame[frame["fold"].isin(folds)].reset_index(drop=True) + outcomes, risk = decide_batch(base_policy, rows, bundle) + floor = rule_floor(base_policy, outcomes) + masks = segment_masks(target.segments, rows) + options, sums = grid(rows, risk, floor, base, masks) + measured[name] = (rows, outcomes, risk, floor, masks, _metrics(sums)) + choice = fit(measured["fit"][5], measured["test"][5], options, base, ids) + + # Before and after, through the real decision code. + test, before_outcomes, risk, floor, masks, _ = measured["test"] + after_outcomes, _ = decide_batch(target, test, bundle) + before = np.array([RANK[o.tier] for o in before_outcomes]) + after = np.array([RANK[o.tier] for o in after_outcomes]) + # The grid's shortcut must agree with the engine for the scales in the file. + replayed = np.maximum(model_rank(risk, base), floor) + for segment, mask in zip(target.segments, masks, strict=True): + replayed[mask] = np.maximum(model_rank(risk[mask], segment.thresholds(base)), floor[mask]) + file_scales = { + s.id: {"warn": s.scale["warn"], "hold": s.scale["hold"]} for s in target.segments + } + + tiers = list(RANK) + moved = Counter( + (tiers[b], tiers[a], "fraud" if y else "legitimate") + for b, a, y in zip(before, after, test["y"].to_numpy(), strict=True) + if a != b + ) + report = { + "model_version": bundle.version, + "base_policy": same_as(base_policy), + "policy_version": target.version, + "fitted_on": list(FIT_FOLDS), + "evaluated_on": "test", + "rows": {"fit": len(measured["fit"][0]), "test": len(test)}, + "days": int(test["day"].nunique()), + "objective": { + "minimise": ( + "the larger of the young-sender and young-receiver false-alert ratios, " + "each against honest payments with no young wallet" + ), + "subject_to": { + "victim_recall_loss_at_most": MAX_RECALL_LOSS, + "taka_recall_loss_at_most": MAX_RECALL_LOSS, + "victim_recall_hold_loss_at_most": MAX_HOLD_RECALL_LOSS, + "guard": "segment warn cut-off at most the model's hold cut-off; hold below 1", + }, + "ties": ( + f"ratios within {GAP_TOLERANCE} count as equal; then the smaller sum of the two " + "ratios, the smaller false-hold ratio, the smaller change" + ), + "warn_scales": list(WARN_SCALES), + "hold_scales": list(HOLD_SCALES), + }, + "thresholds": base, + "segments": [ + { + "id": s.id, + "description": s.description, + "when": [c.model_dump() for c in s.when], + "applies_to": list(s.applies_to), + "scale": s.scale, + "thresholds": s.thresholds(base), + "rows_test": int(mask.sum()), + } + for s, mask in zip(target.segments, masks, strict=True) + ], + "fit": choice, + "policy_matches_fit": all( + abs(file_scales[sid][k] - choice["chosen"]["scales"][sid][k]) < 1e-9 + for sid in ids + for k in ("warn", "hold") + ), + "engine_matches_grid": bool(np.array_equal(replayed, after)), + "tier_counts": { + "before": {t: int((before == RANK[t]).sum()) for t in tiers}, + "after": {t: int((after == RANK[t]).sum()) for t in tiers}, + }, + "tier_changes": [ + {"from": f, "to": t, "label": y, "count": n} for (f, t, y), n in sorted(moved.items()) + ], + "outcomes": {"before": _outcomes(test, before), "after": _outcomes(test, after)}, + "before_after": bootstrap(test, before, after, reps), + "intersectional": intersectional( + test, before, after, pd.read_parquet(data_dir / "wallets.parquet") + ), + } + (model_dir / MITIGATION_FILE).write_text(json.dumps(report, indent=2)) + return report + + +def same_as(policy: Policy) -> str: + """The published policy that decides exactly as this one does, e.g. v2 for v3 without + its segments; otherwise a description.""" + ignore = {"version", "description", "segments"} + for path in sorted(POLICY_DIR.glob("v*.yaml")): + other = load_policy(path.stem) + if not other.segments and other.model_dump(exclude=ignore) == policy.model_dump( + exclude=ignore + ): + return other.version + return f"{policy.version} without segments" + + +def print_summary(report: dict) -> None: + print(f"policy {report['policy_version']} on model {report['model_version']}") + chosen = report["fit"]["chosen"] + print("chosen scales (fitted on validation):", json.dumps(chosen["scales"])) + print("policy file matches the fit:", report["policy_matches_fit"]) + print("engine matches the grid:", report["engine_matches_grid"]) + print(f"{'test period':<58}{'before':>10}{'after':>10}{'diff 95% CI':>24}") + for m in report["before_after"]["metrics"]: + lo, hi = m["difference_ci"] + print(f"{m['label']:<58}{m['before']:>10.5f}{m['after']:>10.5f} [{lo:+.5f}, {hi:+.5f}]") + print("tier counts:", json.dumps(report["tier_counts"])) + print("outcomes:", json.dumps(report["outcomes"])) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fit and evaluate young-wallet segment thresholds") + parser.add_argument("--small", action="store_true") + parser.add_argument("--data", type=Path, default=None) + parser.add_argument("--policy", default=TARGET_POLICY) + parser.add_argument("--model", default=None, help="model version (default: current)") + parser.add_argument("--reps", type=int, default=BOOTSTRAP) + args = parser.parse_args() + data_dir = args.data or settings.data_dir / ("small" if args.small else "full") + print_summary(run(data_dir, args.policy, args.model, reps=args.reps)) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/decision/policies/v3.yaml b/backend/src/fraudlens/decision/policies/v3.yaml new file mode 100644 index 0000000..a3736ee --- /dev/null +++ b/backend/src/fraudlens/decision/policies/v3.yaml @@ -0,0 +1,371 @@ +# FraudLens decision policy. +# +# The models give a risk score; this file says what upay does with it. It is the +# only place business rules live, it is versioned, and every decision records the +# policy version and the rules it went through. Change it by adding a new file, +# not by editing this one. +# +# How each rule was chosen (measured on the validation period, never on test) is +# written up in docs/DECISION_POLICY.md; the segments below in its section 11. + +version: v3 +description: >- + v2 plus cut-offs of their own for payments to or from a wallet under 30 days old, + which v2 interrupts many times more often than others when they are honest. Rules, + including the hard ones, are unchanged and still apply inside a segment. + +# Score cut-offs per tier come from the model version (chosen on validation data to +# meet a precision target within an alert budget). Override one here only as a +# deliberate, reviewed change, for example: overrides: {hold: 0.08} +thresholds: + source: model + overrides: {} + +# A segment scores a group of payments against the cut-offs above times `scale`. +# The first segment that matches decides. A segment changes only the model's tier; +# every rule below still applies afterwards. Guards, applied at load: a segment's +# warn cut-off is capped at the hold cut-off above, so a payment held for anyone +# is at least warned here; its hold cut-off stays below 1, so the hold tier is +# reachable in every segment; cut-offs out of order are refused. A hold scale +# above 1 is the softer tier: a payment between the two hold cut-offs gets step-up +# authentication (the customer re-authenticates and waits) instead of money held +# for an analyst. +segments: + - id: S01_YOUNG_RECEIVER + description: A transfer to a wallet opened less than 30 days ago. + applies_to: [SEND_MONEY] + when: + - {field: r_age_days, op: "<", value: 30} + scale: {warn: 25.0, step_up: 6.0336, hold: 3.0} + fitted: >- + fraudlens.decision.mitigation on val_a + val_b: smallest young-wallet + false-alert ratio losing at most 0.5 pp of victim transfers and money alerted + and 1 pp of victim transfers held. + + - id: S02_YOUNG_SENDER + description: A payment from a wallet opened less than 30 days ago. + when: + - {field: s_age_days, op: "<", value: 30} + scale: {warn: 16.0, step_up: 3.8615, hold: 1.0} + fitted: >- + Same fit as S01, jointly. + +tiers: + allow: + action: proceed + warn: + action: show_warning # the customer sees a warning and can continue + step_up: + action: step_up_auth # extra authentication and a cooling-off wait + cooling_off_minutes: 30 + hold: + action: hold_for_review # money is held until an analyst decides + human_review: true + review_sla_minutes: 30 + +# A rule is: all `when` conditions true -> effect. A condition on a missing value +# is never true. `raise_to` lifts the tier, `cap_at` lowers it; a `hard` rule +# cannot be lowered by a cap. Rules never lower a tier on their own. +rules: + - id: R01_RECIPIENT_CONFIRMED_FRAUD + description: The receiving wallet has been confirmed as fraud by an investigation. + description_bn: তদন্তে প্রাপক ওয়ালেটটি প্রতারণায় জড়িত বলে নিশ্চিত হয়েছে। + applies_to: [SEND_MONEY] + when: + - {field: recipient_flagged, op: "==", value: 1} + effect: raise_to + tier: hold + hard: true + reason: RECIPIENT_CONFIRMED_FRAUD + scenario: scam + + - id: R02_SENDER_CONFIRMED_FRAUD + description: The sending wallet has been confirmed as fraud by an investigation. + description_bn: তদন্তে প্রেরক ওয়ালেটটি প্রতারণায় জড়িত বলে নিশ্চিত হয়েছে। + when: + - {field: sender_flagged, op: "==", value: 1} + effect: raise_to + tier: hold + hard: true + reason: SENDER_CONFIRMED_FRAUD + + - id: R03_FRAUD_HANDSET_NEW_ON_WALLET + description: First transaction from a handset that is new to this wallet and has been used by a confirmed fraud wallet. + description_bn: ওয়ালেটে নতুন এমন একটি হ্যান্ডসেট থেকে প্রথম লেনদেন, যেটি নিশ্চিত প্রতারণার একটি ওয়ালেট ব্যবহার করেছে। + when: + - {field: s_new_device, op: "==", value: 1} + - {field: s_device_flagged, op: "==", value: 1} + effect: raise_to + tier: step_up + reason: SENDER_DEVICE + scenario: takeover + + - id: R04_RECIPIENT_LOOKS_LIKE_MULE + description: The mule model scores the receiving wallet above its alert threshold, whatever the sender's behaviour looks like. + description_bn: প্রেরকের আচরণ যেমনই হোক, মিউল মডেল প্রাপক ওয়ালেটকে সতর্কতা-সীমার উপরে স্কোর দিয়েছে। + applies_to: [SEND_MONEY] + when: + - {field: recipient_mule_alert, op: "==", value: 1} + effect: raise_to + tier: warn + reason: RECIPIENT_MULE_SCORE + + # The behaviour profile. s_place_share is the share of the wallet's earlier + # transactions made from this district or one next to it (home, office and campus + # all count as usual); s_network_share is the same for the network the request + # came over. Both are missing, so these rules say nothing, until the wallet has + # enough history, and the network one also when the channel sends no address. + # Being somewhere new is not suspicious by itself: each rule also needs an amount + # far above what this wallet normally sends. + - id: R05_LARGE_PAYMENT_FROM_UNUSUAL_PLACE + description: An unusually large payment from a place this wallet is not normally used from. + description_bn: ওয়ালেটটি সাধারণত যেখান থেকে ব্যবহৃত হয় না, সেখান থেকে অস্বাভাবিক বড় লেনদেন। + when: + - {field: s_place_share, op: "<=", value: 0.02} + - {field: s_amount_z, op: ">=", value: 3} + effect: raise_to + tier: warn + reason: UNUSUAL_PLACE + scenario: unusual_access + + - id: R06_LARGE_PAYMENT_FROM_UNFAMILIAR_NETWORK + description: An unusually large payment over a network this wallet has never been used from. + description_bn: ওয়ালেটটি আগে কখনো যে নেটওয়ার্ক থেকে ব্যবহৃত হয়নি, সেখান থেকে অস্বাভাবিক বড় লেনদেন। + when: + - {field: s_network_share, op: "==", value: 0} + - {field: s_amount_z, op: ">=", value: 3} + effect: raise_to + tier: warn + reason: UNFAMILIAR_NETWORK + scenario: unusual_access + + - id: R07_LARGE_PAYMENT_FROM_UNUSUAL_PLACE_AND_NETWORK + description: An unusually large payment from an unusual place, over a network this wallet has never been used from. + description_bn: অস্বাভাবিক জায়গা ও আগে কখনো ব্যবহৃত হয়নি এমন নেটওয়ার্ক থেকে অস্বাভাবিক বড় লেনদেন। + when: + - {field: s_place_share, op: "<=", value: 0.02} + - {field: s_network_share, op: "==", value: 0} + - {field: s_amount_z, op: ">=", value: 3} + effect: raise_to + tier: step_up + reason: UNUSUAL_PLACE + scenario: unusual_access + +# Used only when no model score is available (model missing or scoring failed). +# Each signal that is true counts one point. Without a model the system never +# holds money on points alone: the most it does is ask for extra authentication. +fallback: + max_tier: step_up + points: {warn: 4, step_up: 5} + signals: + - id: first_payment_to_receiver + when: [{field: pair_prior_count, op: "==", value: 0}] + - id: receiver_wallet_under_30_days + when: [{field: r_age_days, op: "<", value: 30}] + - id: half_of_balance_or_more + when: [{field: amount_to_balance, op: ">=", value: 0.5}] + - id: first_use_of_handset + when: [{field: s_new_device, op: "==", value: 1}] + - id: amount_5000_or_more + when: [{field: amount, op: ">=", value: 5000}] + - id: receiver_moves_money_on_fast + when: [{field: r_fast_exit_share, op: ">=", value: 0.5}] + - id: handset_on_wallet_under_1_hour + when: [{field: s_device_age_secs, op: "<", value: 3600}] + +# What the customer sees. No scores, no accusation, one clear question. +# A rule that names a scenario gets that wording (the first such rule that fired). +# Otherwise the takeover wording is used when these conditions hold, cash-outs get +# the cash-out wording and transfers the scam wording. +takeover_when: + - {field: s_device_age_secs, op: "<", value: 3600} + +messages: + scam: + warn: + en: >- + Pause before you send. Scammers pose as upay staff, lottery officials or + investment agents and ask people to send money to wallets like this one. + Do you know this person, and did you decide to pay them yourself? + bn: >- + টাকা পাঠানোর আগে একটু থামুন। প্রতারকরা উপায় কর্মকর্তা, লটারি কর্তৃপক্ষ বা + বিনিয়োগ এজেন্ট সেজে এ ধরনের ওয়ালেটে টাকা পাঠাতে বলে। আপনি কি এই ব্যক্তিকে + চেনেন, এবং নিজের সিদ্ধান্তেই টাকা পাঠাচ্ছেন? + step_up: + en: >- + This payment looks like ones that turned out to be scams. To protect you, + please confirm with your PIN again and wait a short time before it is + sent. upay will never ask you to send money to unlock a prize, a refund + or your account. If someone is on the phone telling you what to do, hang up. + bn: >- + এই লেনদেনটি আগে প্রতারণা বলে প্রমাণিত লেনদেনগুলোর মতো। আপনার সুরক্ষার জন্য + আবার পিন দিয়ে নিশ্চিত করুন এবং পাঠানোর আগে কিছুক্ষণ অপেক্ষা করুন। পুরস্কার, + রিফান্ড বা অ্যাকাউন্ট চালু করার জন্য উপায় কখনো টাকা পাঠাতে বলে না। কেউ ফোনে + নির্দেশ দিতে থাকলে ফোন কেটে দিন। + hold: + en: >- + We have paused this payment to check it, because it closely matches known + scams. Your money is still in your wallet. A upay officer will review it + shortly and may call you from our official number. You do not need to do + anything, and you should not share your PIN or OTP with anyone. + bn: >- + পরিচিত প্রতারণার সাথে মিল থাকায় আমরা এই লেনদেনটি যাচাইয়ের জন্য স্থগিত + রেখেছি। আপনার টাকা আপনার ওয়ালেটেই আছে। একজন উপায় কর্মকর্তা শীঘ্রই এটি + পর্যালোচনা করবেন এবং আমাদের অফিসিয়াল নম্বর থেকে আপনাকে ফোন করতে পারেন। + আপনাকে কিছু করতে হবে না; পিন বা ওটিপি কাউকে দেবেন না। + takeover: + warn: + en: >- + This payment is being made from a phone we have not seen on your wallet + for long. If this is not you, do not continue and change your PIN now. + bn: >- + এই লেনদেনটি এমন একটি ফোন থেকে করা হচ্ছে যা আপনার ওয়ালেটে অল্প সময় ধরে + ব্যবহৃত হচ্ছে। এটি আপনি না হলে লেনদেন বন্ধ করুন এবং এখনই পিন পরিবর্তন করুন। + step_up: + en: >- + We need to make sure this is really you. This payment comes from a phone + that is new to your wallet. Please verify your identity to continue. + bn: >- + আমাদের নিশ্চিত হতে হবে যে এটি সত্যিই আপনি। এই লেনদেনটি আপনার ওয়ালেটে নতুন + একটি ফোন থেকে আসছে। চালিয়ে যেতে আপনার পরিচয় যাচাই করুন। + hold: + en: >- + We have paused this payment because your wallet is being used from a new + phone in an unusual way. Your money is still in your wallet. A upay + officer will review it shortly and contact you on your registered number. + bn: >- + আপনার ওয়ালেট একটি নতুন ফোন থেকে অস্বাভাবিকভাবে ব্যবহৃত হওয়ায় আমরা এই + লেনদেনটি স্থগিত রেখেছি। আপনার টাকা আপনার ওয়ালেটেই আছে। একজন উপায় + কর্মকর্তা শীঘ্রই এটি পর্যালোচনা করে আপনার নিবন্ধিত নম্বরে যোগাযোগ করবেন। + unusual_access: + warn: + en: >- + This is a large payment from a place or network where your wallet is not + normally used. If this is not you, do not continue and change your PIN now. + bn: >- + এটি একটি বড় অঙ্কের লেনদেন, যা এমন জায়গা বা নেটওয়ার্ক থেকে করা হচ্ছে যেখান + থেকে আপনার ওয়ালেট সাধারণত ব্যবহৃত হয় না। এটি আপনি না হলে লেনদেন বন্ধ করুন + এবং এখনই পিন পরিবর্তন করুন। + step_up: + en: >- + We need to make sure this is really you. This large payment comes from a + place or a network where your wallet is not normally used. Please verify + your identity to continue. + bn: >- + আমাদের নিশ্চিত হতে হবে যে এটি সত্যিই আপনি। এই বড় অঙ্কের লেনদেনটি এমন জায়গা বা + নেটওয়ার্ক থেকে আসছে যেখান থেকে আপনার ওয়ালেট সাধারণত ব্যবহৃত হয় না। চালিয়ে + যেতে আপনার পরিচয় যাচাই করুন। + hold: + en: >- + We have paused this payment because your wallet is being used in an + unusual way, from somewhere it is not normally used. Your money is still + in your wallet. A upay officer will review it shortly and contact you on + your registered number. + bn: >- + আপনার ওয়ালেট সাধারণত যেখান থেকে ব্যবহৃত হয় না এমন জায়গা থেকে অস্বাভাবিকভাবে + ব্যবহৃত হওয়ায় আমরা এই লেনদেনটি স্থগিত রেখেছি। আপনার টাকা আপনার ওয়ালেটেই + আছে। একজন উপায় কর্মকর্তা শীঘ্রই এটি পর্যালোচনা করে আপনার নিবন্ধিত নম্বরে + যোগাযোগ করবেন। + cash_out: + warn: + en: >- + Please check this cash-out. If someone asked you to withdraw money that + another person sent you, it may be part of a scam. + bn: >- + এই ক্যাশ-আউটটি যাচাই করে নিন। অন্য কারও পাঠানো টাকা তুলে দিতে কেউ আপনাকে + বলে থাকলে এটি প্রতারণার অংশ হতে পারে। + step_up: + en: >- + This cash-out needs an extra check. Please verify your identity to continue. + bn: >- + এই ক্যাশ-আউটের জন্য বাড়তি যাচাই প্রয়োজন। চালিয়ে যেতে আপনার পরিচয় যাচাই করুন। + hold: + en: >- + We have paused this cash-out to check it. The money is still in the + wallet. A upay officer will review it shortly. + bn: >- + যাচাইয়ের জন্য আমরা এই ক্যাশ-আউটটি স্থগিত রেখেছি। টাকা ওয়ালেটেই আছে। একজন + উপায় কর্মকর্তা শীঘ্রই এটি পর্যালোচনা করবেন। + +# Next steps offered to the analyst. These are suggestions for a person; the +# system does not carry any of them out by itself. +recommendations: + - id: CALL_SENDER + min_tier: step_up + applies_to: [SEND_MONEY] + en: Call the sender on their registered number and ask who asked them to make this payment. Do not use a number given during the transaction. + bn: প্রেরককে তার নিবন্ধিত নম্বরে ফোন করে জানুন কে তাকে এই টাকা পাঠাতে বলেছে। লেনদেনের সময় দেওয়া কোনো নম্বর ব্যবহার করবেন না। + + - id: FREEZE_RECIPIENT_REVIEW + min_tier: step_up + applies_to: [SEND_MONEY] + when: + - {field: recipient_mule_alert, op: "==", value: 1} + en: Open a freeze review on the receiving wallet. A freeze takes effect only after a second analyst approves it. + bn: প্রাপক ওয়ালেটের জন্য ফ্রিজ পর্যালোচনা শুরু করুন। দ্বিতীয় একজন বিশ্লেষক অনুমোদন করলে তবেই ফ্রিজ কার্যকর হবে। + needs_second_approver: true + + - id: CHECK_OTHER_SENDERS + min_tier: step_up + applies_to: [SEND_MONEY] + when: + - {field: r_fan_in_7d, op: ">=", value: 3} + en: Check the other wallets that paid this receiver in the last 7 days. Some may be victims who have not reported yet. + bn: গত ৭ দিনে এই প্রাপককে টাকা পাঠানো অন্য ওয়ালেটগুলো দেখুন। তাদের কেউ কেউ এখনো অভিযোগ না করা ভুক্তভোগী হতে পারেন। + + - id: SECURE_ACCOUNT + min_tier: step_up + when: + - {field: s_device_age_secs, op: "<", value: 3600} + en: Treat as a possible account takeover. Confirm the handset change with the customer, then reset the PIN and sign out other devices. + bn: সম্ভাব্য অ্যাকাউন্ট দখল হিসেবে বিবেচনা করুন। গ্রাহকের সাথে হ্যান্ডসেট পরিবর্তন নিশ্চিত করুন, তারপর পিন রিসেট করে অন্য ডিভাইস থেকে সাইন আউট করুন। + + - id: CONFIRM_PLACE + min_tier: step_up + when: + - {field: s_place_share, op: "<=", value: 0.02} + en: Ask the customer, on their registered number, whether they are travelling. + bn: গ্রাহককে তার নিবন্ধিত নম্বরে ফোন করে জানুন তিনি ভ্রমণে আছেন কি না। + + - id: CONFIRM_NETWORK + min_tier: step_up + when: + - {field: s_network_share, op: "==", value: 0} + en: Ask the customer whether they changed their internet connection or SIM. + bn: গ্রাহক তার ইন্টারনেট সংযোগ বা সিম পরিবর্তন করেছেন কি না জিজ্ঞাসা করুন। + + - id: REVIEW_SENDER_HANDSET + min_tier: warn + when: + - {field: s_device_flagged, op: "==", value: 1} + en: Review every wallet that has used the sender's handset. It has been used by a confirmed fraud wallet. + bn: প্রেরকের হ্যান্ডসেট ব্যবহার করা সব ওয়ালেট পর্যালোচনা করুন। এটি নিশ্চিত প্রতারণার একটি ওয়ালেট ব্যবহার করেছে। + + - id: REVIEW_RECIPIENT_HANDSET + min_tier: warn + applies_to: [SEND_MONEY] + when: + - {field: r_device_flagged, op: "==", value: 1} + en: Review every wallet that shares the receiver's handset. It has been used by a confirmed fraud wallet. + bn: প্রাপকের হ্যান্ডসেট ব্যবহার করা সব ওয়ালেট পর্যালোচনা করুন। এটি নিশ্চিত প্রতারণার একটি ওয়ালেট ব্যবহার করেছে। + + - id: TRACE_INCOMING_FUNDS + min_tier: step_up + applies_to: [CASH_OUT] + when: + - {field: s_secs_since_last_in, op: "<", value: 1800} + en: Trace the transfers this wallet received just before the cash-out and contact those senders. This may be scam money being withdrawn. + bn: ক্যাশ-আউটের ঠিক আগে এই ওয়ালেটে আসা টাকাগুলো খুঁজে দেখুন এবং সেই প্রেরকদের সাথে যোগাযোগ করুন। এটি প্রতারণার টাকা তুলে নেওয়া হতে পারে। + + - id: REVIEW_AGENT + min_tier: step_up + applies_to: [CASH_OUT] + when: + - {field: a_flagged_customers, op: ">=", value: 1} + en: Review this agent's recent cash-outs. Some of the agent's customers were confirmed as fraud. + bn: এই এজেন্টের সাম্প্রতিক ক্যাশ-আউটগুলো পর্যালোচনা করুন। এজেন্টের কিছু গ্রাহক প্রতারণায় জড়িত বলে নিশ্চিত হয়েছে। + + - id: RECORD_OUTCOME + min_tier: hold + en: Decide within the review time, then record the outcome. If the customer confirms the payment and the receiver checks out, release the hold. + bn: পর্যালোচনার সময়ের মধ্যে সিদ্ধান্ত নিন এবং ফলাফল লিখে রাখুন। গ্রাহক লেনদেন নিশ্চিত করলে এবং প্রাপক সম্পর্কে সন্দেহ না থাকলে হোল্ড তুলে নিন। diff --git a/backend/src/fraudlens/decision/policy.py b/backend/src/fraudlens/decision/policy.py index 5110660..6e8077d 100644 --- a/backend/src/fraudlens/decision/policy.py +++ b/backend/src/fraudlens/decision/policy.py @@ -124,6 +124,60 @@ class Thresholds(_Model): overrides: dict[Literal["warn", "step_up", "hold"], float] = Field(default_factory=dict) +class Segment(_Model): + """A group of payments scored against its own cut-offs, e.g. to a wallet under 30 days old. + + The cut-offs are the model's own times `scale`, so they follow the model version + they are resolved against. A segment changes only the model's tier: rules are + applied afterwards exactly as for any other payment, so a hard rule still holds. + Guards, applied when the thresholds are resolved: the segment's warn cut-off is + capped at the policy's hold cut-off, so a payment held for anyone is at least + warned in every segment; the hold cut-off stays below 1, so every segment can + still be held for a person; and a file whose cut-offs still break + `0 < warn <= step_up <= hold < 1` (a scale below 1) is refused. + """ + + id: str = Field(pattern=r"^[A-Z][A-Z0-9_]*$") + description: str = Field(min_length=1) + applies_to: tuple[str, ...] = SCORED_TYPES + when: tuple[Condition, ...] = Field(min_length=1) + scale: dict[Literal["warn", "step_up", "hold"], float] + # How the scale was chosen, so the file says where its numbers come from. + fitted: str = "" + + @model_validator(mode="after") + def _consistent(self) -> Segment: + if unknown := set(self.applies_to) - set(SCORED_TYPES): + raise ValueError(f"segment {self.id}: unknown transaction types {sorted(unknown)}") + if set(self.scale) != set(ALERT_TIERS): + raise ValueError(f"segment {self.id}: scale needs each of {list(ALERT_TIERS)}") + if not all(0 < value < 1_000 for value in self.scale.values()): + raise ValueError(f"segment {self.id}: scales must be positive and below 1000") + return self + + def thresholds(self, base: Mapping[str, float]) -> dict[str, float]: + """This segment's cut-offs, given the policy's resolved ones; refused if unsafe.""" + scaled = {tier: float(base[tier]) * self.scale[tier] for tier in ALERT_TIERS} + # The same scales are resolved against every model version (a shadow model uses + # the served policy), and their spacing differs. So rather than the file being + # refused for one model and accepted for another: the warn cut-off is capped at + # the policy's hold cut-off; the hold cut-off keeps at least 1/scale of the + # policy's room below 1 (so something can always be held); step-up stays between. + scaled["warn"] = min(scaled["warn"], float(base["hold"])) + if self.scale["hold"] > 1: + room = (1 - float(base["hold"])) / self.scale["hold"] + scaled["hold"] = min(scaled["hold"], 1 - room) + scaled["step_up"] = min(max(scaled["step_up"], scaled["warn"]), scaled["hold"]) + try: + check_thresholds(scaled) + except PolicyError as exc: + raise PolicyError(f"segment {self.id}: {exc}") from exc + return scaled + + def matches(self, txn_type: str, fields: Mapping[str, float]) -> bool: + return txn_type in self.applies_to and evaluate(self.when, fields) == "fired" + + class Signal(_Model): id: str when: tuple[Condition, ...] = Field(min_length=1) @@ -169,6 +223,8 @@ class Policy(_Model): version: str = Field(min_length=1) description: str = "" thresholds: Thresholds = Thresholds() + # Payments scored against their own cut-offs; the first segment that matches applies. + segments: tuple[Segment, ...] = () tiers: dict[Tier, TierSpec] rules: tuple[Rule, ...] fallback: Fallback @@ -188,6 +244,9 @@ def _complete(self) -> Policy: ids = [r.id for r in self.rules] if len(ids) != len(set(ids)): raise ValueError("rule ids must be unique") + segment_ids = [s.id for s in self.segments] + if len(segment_ids) != len(set(segment_ids)): + raise ValueError("segment ids must be unique") rec_ids = [r.id for r in self.recommendations] if len(rec_ids) != len(set(rec_ids)): raise ValueError("recommendation ids must be unique") @@ -215,8 +274,13 @@ def resolve_thresholds(self, manifest: Mapping | None) -> dict[str, float]: base = {tier: float(manifest["thresholds"][tier]) for tier in ALERT_TIERS} resolved = base | dict(self.thresholds.overrides) check_thresholds(resolved) + for segment in self.segments: # refuse an unsafe segment now, not at the first payment + segment.thresholds(resolved) return resolved + def segment_for(self, txn_type: str, fields: Mapping[str, float]) -> Segment | None: + return next((s for s in self.segments if s.matches(txn_type, fields)), None) + def check_thresholds(thresholds: Mapping[str, float]) -> None: warn, step_up, hold = (thresholds[t] for t in ALERT_TIERS) @@ -257,6 +321,8 @@ class Outcome: trace: tuple[dict, ...] fired: tuple[Rule, ...] fallback_signals: tuple[str, ...] = () + segment: Segment | None = None # the segment whose cut-offs set the model's tier + thresholds: Mapping[str, float] | None = None # the cut-offs actually used def apply_policy( @@ -274,6 +340,7 @@ def apply_policy( if txn_type not in SCORED_TYPES: raise ValueError(f"{txn_type} transactions are not scored") signals: tuple[str, ...] = () + segment = None if risk is None or thresholds is None: mode, model_tier = "rules_only", None fallback = policy.fallback @@ -290,6 +357,9 @@ def apply_policy( if math.isnan(risk): raise ValueError("risk is NaN") mode, decided_by = "model", "model" + # A segment moves the model's cut-offs only; the rules below see every payment alike. + if segment := policy.segment_for(txn_type, fields): + thresholds = segment.thresholds(thresholds) tier = model_tier = tier_for(risk, thresholds) trace, fired = [], [] @@ -325,7 +395,9 @@ def apply_policy( if rule.effect == "cap_at" and RANK[rule.tier] < RANK[tier]: tier, decided_by = rule.tier, f"cap:{rule.id}" - return Outcome(tier, model_tier, mode, decided_by, tuple(trace), tuple(fired), signals) + return Outcome( + tier, model_tier, mode, decided_by, tuple(trace), tuple(fired), signals, segment, thresholds + ) def _plain(value: float | None) -> float | None: diff --git a/backend/src/fraudlens/external/__init__.py b/backend/src/fraudlens/external/__init__.py new file mode 100644 index 0000000..83bc3d8 --- /dev/null +++ b/backend/src/fraudlens/external/__init__.py @@ -0,0 +1,4 @@ +"""External validation: the FraudLens modelling approach on public fraud datasets. + +See docs/MODEL_CARD.md, "External validation". `make external` runs all of it. +""" diff --git a/backend/src/fraudlens/external/baf.py b/backend/src/fraudlens/external/baf.py new file mode 100644 index 0000000..37b9ad5 --- /dev/null +++ b/backend/src/fraudlens/external/baf.py @@ -0,0 +1,173 @@ +"""Bank Account Fraud (BAF, NeurIPS 2022), Base variant: the FraudLens approach on it. + +BAF is generated from a real bank's account-opening data by a privacy-preserving +tabular generator: 1,000,000 applications over 8 months, 1.1% fraudulent. It is not +payments data. The closest FraudLens task is the mule-wallet model: spotting a +wallet that was opened to receive scam money. What maps onto FraudLens ideas: + +- velocity: applications in the last 6 hours, 24 hours and 4 weeks + (`velocity_6h`, `velocity_24h`, `velocity_4w`), applications from the same + postcode and branch (`zip_count_4w`, `bank_branch_count_8w`); +- shared devices and identities (FraudLens' shared-handset signal): + `device_distinct_emails_8w`, `date_of_birth_distinct_emails_4w`; +- account age and history: `bank_months_count`, `prev_address_months_count`. + +Temporal split as in the benchmark (months 0-5 train, 6-7 test), with month 5 +held out of training to stop boosting and fix thresholds. The benchmark's own +metric is recall at 5% false-positive rate; 0.5% and 1% are reported too, because +those are the alert rates FraudLens works at. Every remaining column is used as +given; categorical columns become integer codes fitted on the training months. + + uv run python -m fraudlens.external.baf +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from ..config import settings +from . import download +from .evaluate import ( + anomaly_score, + bootstrap_ci, + finite, + fit_isolation_forest, + fit_lightgbm, + fit_logistic, + judge, + top_gain, +) + +TARGET = "fraud_bool" +TRAIN_LAST_MONTH, VAL_MONTH = 4, 5 # test: months 6 and 7 +FPRS = (0.005, 0.01, 0.05) +CI_FPR = 0.05 # the benchmark's metric +VELOCITY_FEATURES = ("velocity_6h", "velocity_24h", "velocity_4w") + + +def load(path: Path) -> pd.DataFrame: + df = pd.read_parquet(path) + df[TARGET] = df[TARGET].astype(int) + return df + + +def encode(df: pd.DataFrame) -> tuple[pd.DataFrame, list[str]]: + """Integer codes for text columns, from the training months' categories.""" + out = df.copy() + train = df["month"] <= TRAIN_LAST_MONTH + for col in df.columns: + if col != TARGET and not pd.api.types.is_numeric_dtype(df[col]): + categories = sorted(df.loc[train, col].astype(str).unique()) + codes = {value: i for i, value in enumerate(categories)} + out[col] = df[col].astype(str).map(codes).fillna(-1).astype(int) + features = [c for c in out.columns if c not in (TARGET, "month")] + return out, features + + +def evaluate(df: pd.DataFrame, replicates: int) -> dict: + data, features = encode(df) + folds = { + "train": data[data["month"] <= TRAIN_LAST_MONTH], + "val": data[data["month"] == VAL_MONTH], + "test": data[data["month"] > VAL_MONTH], + } + y = {k: v[TARGET].to_numpy() for k, v in folds.items()} + scores: dict[str, tuple[np.ndarray, np.ndarray]] = {} + + # A velocity-only rule: the sum of the ranks of the three velocity counts. + def velocity_rank(part: pd.DataFrame) -> np.ndarray: + ref = folds["train"] + return sum( + np.searchsorted(np.sort(ref[c].to_numpy()), part[c].to_numpy()) / len(ref) + for c in VELOCITY_FEATURES + ) + + scores["velocity_rule"] = tuple(velocity_rank(folds[k]) for k in ("val", "test")) + scores["credit_risk_score_only"] = tuple( + folds[k]["credit_risk_score"].to_numpy(np.float64) for k in ("val", "test") + ) + logistic = fit_logistic(folds["train"][features], y["train"]) + scores["logistic_regression"] = tuple( + logistic.predict_proba(finite(folds[k][features]))[:, 1] for k in ("val", "test") + ) + forest = fit_isolation_forest(folds["train"][features]) + scores["isolation_forest"] = tuple( + anomaly_score(forest, folds[k][features]) for k in ("val", "test") + ) + booster = fit_lightgbm(folds["train"][features], y["train"], folds["val"][features], y["val"]) + scores["lightgbm"] = tuple( + booster.predict(folds[k][features].to_numpy(np.float64)) for k in ("val", "test") + ) + + results = { + name: judge(y["val"], s_val, y["test"], s_test, FPRS) + for name, (s_val, s_test) in scores.items() + } + s_val, s_test = scores["lightgbm"] + # No time finer than a month: applications are resampled one by one. + results["lightgbm"]["bootstrap_by_application"] = bootstrap_ci( + y["val"], s_val, y["test"], s_test, np.arange(len(s_test)), CI_FPR, replicates + ) + results["lightgbm"]["trees"] = booster.num_trees() + results["lightgbm"]["top_features"] = top_gain(booster) + + return { + "dataset": "Bank Account Fraud, Base (Jesus et al., NeurIPS 2022)", + "source": download.SOURCES["baf"].url, + "sha256": download.SOURCES["baf"].sha256, + "rows_total": int(len(df)), + "split_by_month": {"train": "0-4", "val": "5", "test": "6-7"}, + "folds": { + k: { + "rows": int(len(v)), + "fraud": int(v[TARGET].sum()), + "base_rate": round(float(v[TARGET].mean()), 5), + } + for k, v in folds.items() + }, + "features": features, + "thresholds": "fixed on val at each target FPR, applied unchanged to test", + "models": results, + } + + +def run(path: Path | None = None, out_dir: Path | None = None, replicates: int = 500) -> dict: + t0 = time.perf_counter() + report = evaluate(load(path or download.fetch("baf")), replicates) + report["seconds"] = round(time.perf_counter() - t0, 1) + out = out_dir or settings.artifacts_dir / "external" + out.mkdir(parents=True, exist_ok=True) + (out / "baf.json").write_text(json.dumps(report, indent=2)) + return report + + +def print_summary(report: dict) -> None: + print(f"\n{report['dataset']}: folds {json.dumps(report['folds'])}") + rows = { + name: { + "pr_auc": m["pr_auc"], + "roc_auc": m["roc_auc"], + **{f"recall@{f:g}": m[f"at_fpr_{f:g}"]["recall"] for f in FPRS}, + **{f"testfpr@{f:g}": m[f"at_fpr_{f:g}"]["fpr_on_test"] for f in FPRS}, + } + for name, m in report["models"].items() + } + print(pd.DataFrame(rows).T.to_string()) + print("lightgbm", json.dumps(report["models"]["lightgbm"]["bootstrap_by_application"])) + print(" top features:", report["models"]["lightgbm"]["top_features"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="FraudLens approach on BAF") + parser.add_argument("--replicates", type=int, default=500) + print_summary(run(replicates=parser.parse_args().replicates)) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/external/download.py b/backend/src/fraudlens/external/download.py new file mode 100644 index 0000000..47e282f --- /dev/null +++ b/backend/src/fraudlens/external/download.py @@ -0,0 +1,123 @@ +"""Download the public datasets used for external validation, and verify them. + + uv run python -m fraudlens.external.download # both + uv run python -m fraudlens.external.download paysim + +Files land in backend/data/external// (gitignored). A file already present +with the right checksum is not downloaded again; a partial file is resumed; a file +with the wrong checksum is an error, never silently used. + +Sources (no account needed): + +- PaySim: the Kaggle file (ealaxi/paysim1, "PS_20174392719_1491204439457_log.csv") + as mirrored on Hugging Face. The SHA-256 below is the one Hugging Face publishes + for the LFS object; it has 6,362,620 rows, the size of the original release. +- Bank Account Fraud, Base variant (Jesus et al., NeurIPS 2022): the OpenML copy + (dataset 46793, "BAF_base", version 2) as parquet. OpenML publishes no SHA-256 + for the parquet file, so the value below is the one measured on first download + (2026-10-07); the content check (1,000,000 rows, 11,029 frauds) matches the paper. +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from ..config import settings + + +@dataclass(frozen=True) +class Source: + name: str + url: str + file: str + sha256: str + size: int + citation: str + + +SOURCES = { + "paysim": Source( + name="paysim", + url="https://huggingface.co/datasets/theman10/paysim/resolve/main/paysim.csv", + file="paysim.csv", + sha256="16910f90577b0d981bf8ff289714510bb89bc71bff7d3f220f024e287e4eea6b", + size=493_534_783, + citation=( + "E. A. Lopez-Rojas, A. Elmir, S. Axelsson. PaySim: A financial mobile money " + "simulator for fraud detection. EMSS 2016." + ), + ), + "baf": Source( + name="baf", + url="https://data.openml.org/datasets/0004/46793/dataset_46793.pq", + file="baf_base.parquet", + sha256="217e2fbd62a0f501c05ece5560841bdba6f4d7a3d302383c662c4d9efb22a872", + size=69_498_475, + citation=( + "S. Jesus et al. Turning the Tables: Biased, Imbalanced, Dynamic Tabular " + "Datasets for ML Evaluation. NeurIPS 2022 Datasets and Benchmarks." + ), + ), +} + + +def external_dir() -> Path: + return settings.data_dir / "external" + + +def path_of(name: str) -> Path: + source = SOURCES[name] + return external_dir() / source.name / source.file + + +def sha256_of(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def fetch(name: str) -> Path: + """The verified local file, downloading or resuming it if needed.""" + source = SOURCES[name] + target = path_of(name) + target.parent.mkdir(parents=True, exist_ok=True) + if not (target.exists() and target.stat().st_size == source.size): + if shutil.which("curl") is None: + raise RuntimeError("curl is needed to download the external datasets") + print(f"downloading {name} ({source.size / 1e6:.0f} MB) from {source.url}") + for _ in range(5): # the mirrors sometimes close long transfers; resume + subprocess.run( + ["curl", "-sSL", "--retry", "3", "-C", "-", "-o", str(target), source.url], + check=False, + ) + if target.exists() and target.stat().st_size >= source.size: + break + actual = sha256_of(target) + if actual != source.sha256: + raise RuntimeError( + f"{target}: SHA-256 {actual} does not match the expected {source.sha256}. " + "Delete the file and run again; if it persists the mirror has changed." + ) + print(f"{name}: {target} ok (sha256 {actual[:12]}...)") + return target + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download and verify the external datasets") + parser.add_argument("names", nargs="*", help=f"any of {', '.join(SOURCES)}; default: all") + names = parser.parse_args().names or list(SOURCES) + if unknown := set(names) - set(SOURCES): + parser.error(f"unknown dataset: {', '.join(sorted(unknown))}") + for name in names: + fetch(name) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/external/evaluate.py b/backend/src/fraudlens/external/evaluate.py new file mode 100644 index 0000000..2764bc0 --- /dev/null +++ b/backend/src/fraudlens/external/evaluate.py @@ -0,0 +1,140 @@ +"""Shared fitting and scoring for the external datasets. + +Every score is judged the same way: thresholds are fixed on the validation period +at a target false-positive rate, then recall and the false-positive rate actually +reached are measured on the later test period. PR-AUC and ROC-AUC are threshold-free. +""" + +from __future__ import annotations + +import lightgbm as lgb +import numpy as np +import pandas as pd +from sklearn.ensemble import IsolationForest +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import average_precision_score, roc_auc_score +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +from ..models.train import LGB_PARAMS, SEED + +BOOTSTRAP_REPLICATES = 500 + + +def threshold_at_fpr(y: np.ndarray, score: np.ndarray, fpr: float) -> float: + """Lowest threshold whose false-positive rate on (y, score) is at most `fpr`.""" + negatives = np.sort(score[y == 0])[::-1] + k = int(np.floor(fpr * len(negatives))) # negatives allowed at or above the threshold + if k >= len(negatives): + return float(negatives[-1]) + # Alert on score > negatives[k]: exactly the k highest negatives (ties go below). + return float(np.nextafter(negatives[k], np.inf)) + + +def judge( + y_val: np.ndarray, + s_val: np.ndarray, + y_test: np.ndarray, + s_test: np.ndarray, + fprs: tuple[float, ...], +) -> dict: + out = { + "pr_auc": round(float(average_precision_score(y_test, s_test)), 4), + "roc_auc": round(float(roc_auc_score(y_test, s_test)), 4), + } + for fpr in fprs: + thr = threshold_at_fpr(y_val, s_val, fpr) + flagged = s_test >= thr + # The literature (and the BAF benchmark) usually sets the threshold on the test + # set itself; reported separately, because no deployment can do that. + exact = s_test >= threshold_at_fpr(y_test, s_test, fpr) + out[f"at_fpr_{fpr:g}"] = { + "recall": round(float(flagged[y_test == 1].mean()), 4), + "fpr_on_test": round(float(flagged[y_test == 0].mean()), 5), + "precision": round(float(y_test[flagged].mean()), 4) if flagged.any() else None, + "recall_threshold_set_on_test": round(float(exact[y_test == 1].mean()), 4), + } + return out + + +def bootstrap_ci( + y_val: np.ndarray, + s_val: np.ndarray, + y_test: np.ndarray, + s_test: np.ndarray, + units: np.ndarray, + fpr: float, + replicates: int = BOOTSTRAP_REPLICATES, + seed: int = SEED, +) -> dict: + """Percentile 95% intervals for test recall at `fpr` and PR-AUC. + + `units` are resampled with replacement (e.g. hours, so that transactions of the + same hour stay together). The validation threshold is held fixed. + """ + thr = threshold_at_fpr(y_val, s_val, fpr) + flagged = s_test >= thr + codes, inverse = np.unique(units, return_inverse=True) + rng = np.random.default_rng(seed) + recall, pr_auc = [], [] + pos = y_test == 1 + for _ in range(replicates): + counts = np.bincount(rng.integers(0, len(codes), len(codes)), minlength=len(codes)) + w = counts[inverse].astype(np.float64) + recall.append((w * (flagged & pos)).sum() / max((w * pos).sum(), 1e-12)) + keep = w > 0 + pr_auc.append(average_precision_score(y_test[keep], s_test[keep], sample_weight=w[keep])) + + def ci(values: list[float]) -> list[float]: + return [round(float(np.quantile(values, q)), 4) for q in (0.025, 0.975)] + + return { + "units": int(len(codes)), + "replicates": replicates, + f"recall_at_fpr_{fpr:g}": ci(recall), + "pr_auc": ci(pr_auc), + } + + +def fit_lightgbm( + x_train: pd.DataFrame, y_train: np.ndarray, x_val: pd.DataFrame, y_val: np.ndarray +) -> lgb.Booster: + """The FraudLens transaction-model recipe: same parameters, early stopping on val.""" + cols = list(x_train.columns) + dtrain = lgb.Dataset(x_train.to_numpy(np.float64), y_train, feature_name=cols) + dval = lgb.Dataset(x_val.to_numpy(np.float64), y_val, reference=dtrain) + booster = lgb.train( + LGB_PARAMS, + dtrain, + num_boost_round=2000, + valid_sets=[dval], + callbacks=[lgb.early_stopping(100, verbose=False)], + ) + return lgb.Booster(model_str=booster.model_to_string(num_iteration=booster.best_iteration)) + + +def fit_logistic(x_train: pd.DataFrame, y_train: np.ndarray): + model = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000, C=1.0)) + return model.fit(finite(x_train), y_train) + + +def fit_isolation_forest(x_train: pd.DataFrame) -> IsolationForest: + sample = x_train.sample(min(len(x_train), 200_000), random_state=SEED) + return IsolationForest(n_estimators=100, random_state=SEED, n_jobs=-1).fit(finite(sample)) + + +def anomaly_score(model: IsolationForest, x: pd.DataFrame) -> np.ndarray: + return -model.score_samples(finite(x)) + + +def finite(x: pd.DataFrame) -> np.ndarray: + """Missing values (e.g. no earlier transfer) as -1, which no real value takes.""" + return np.nan_to_num(x.to_numpy(np.float64), nan=-1.0, posinf=-1.0, neginf=-1.0) + + +def top_gain(booster: lgb.Booster, n: int = 8) -> list[dict]: + gain = booster.feature_importance("gain") + share = gain / gain.sum() if gain.sum() else gain + order = np.argsort(-share)[:n] + names = booster.feature_name() + return [{"feature": names[i], "gain_share": round(float(share[i]), 3)} for i in order] diff --git a/backend/src/fraudlens/external/paysim.py b/backend/src/fraudlens/external/paysim.py new file mode 100644 index 0000000..91185f3 --- /dev/null +++ b/backend/src/fraudlens/external/paysim.py @@ -0,0 +1,275 @@ +"""PaySim: the FraudLens approach on an external mobile-money simulation. + +PaySim (Lopez-Rojas et al., 2016) was calibrated on a month of logs from a real +mobile-money service in an African country. One `step` is one hour (743 steps). +Fraud there is account takeover: the fraudster TRANSFERs the victim's balance to +another account and CASH_OUTs. Only TRANSFER and CASH_OUT rows are ever fraud, so +those are the rows scored, as in most work on PaySim. + +What maps onto FraudLens features, computed only from earlier rows (file order +within an hour is taken as time order): + +- receiver-centric: earlier incoming payments to the receiving account (all time + and last 24 hours), their value, hours since it last received, hours since it was + first seen, earlier incoming TRANSFERs, amount against its usual incoming amount; +- velocity: the 24-hour counts above; the sender's earlier payments; +- cash-out timing: hours since the *sender* last received money, and the amount + against what it received (FraudLens' "fast exit"). In PaySim the account that + receives a fraudulent TRANSFER is never the one that cashes out (0 of 4,097), so + this cannot see the fraud chain; it is kept because the mapping is the honest one. + +Not mapped (no equivalent in PaySim): devices, agents, districts, wallet age, +confirmed-fraud flags, scam typologies, social engineering. + +Balance fields are a known PaySim artefact: 97.8% of frauds move exactly the +sender's whole balance and no legitimate TRANSFER or CASH_OUT does. The main model +leaves balances out; a second model adds the pre-transaction balances, to show the +size of the artefact. Post-transaction balances are never used. + + uv run python -m fraudlens.external.paysim +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from ..config import settings +from . import download +from .evaluate import ( + anomaly_score, + bootstrap_ci, + finite, + fit_isolation_forest, + fit_lightgbm, + fit_logistic, + judge, + top_gain, +) + +SCORED_TYPES = ("TRANSFER", "CASH_OUT") +TRAIN_LAST_STEP, VAL_LAST_STEP = 300, 400 # train 1-300, val 301-400, test 401-743 +FPRS = (0.005, 0.01, 0.05) +CI_FPR = 0.005 # the FraudLens headline false-alert rate is about 0.5% +WINDOW = 24 # hours + +BEHAVIOUR_FEATURES = ( + "amount", + "is_transfer", + "hour_of_day", + "r_prior_in", + "r_prior_in_24h", + "r_prior_in_value_24h", + "r_hours_since_last_in", + "r_hours_since_first_seen", + "r_prior_in_transfers", + "amount_vs_r_mean_in", + "s_prior_out", + "s_hours_since_received", + "amount_vs_s_received", +) +BALANCE_FEATURES = ("oldbalanceOrg", "oldbalanceDest", "amount_vs_balance") + + +def load(path: Path) -> pd.DataFrame: + df = pd.read_csv( + path, + usecols=[ + "step", + "type", + "amount", + "nameOrig", + "oldbalanceOrg", + "nameDest", + "oldbalanceDest", + "isFraud", + "isFlaggedFraud", + ], + ) + return df.sort_values("step", kind="stable").reset_index(drop=True) + + +def _prefix(values: np.ndarray) -> np.ndarray: + return np.concatenate([[0.0], np.cumsum(values, dtype=np.float64)]) + + +def add_features(df: pd.DataFrame) -> pd.DataFrame: + """Causal features: each row sees only rows before it (earlier hour or earlier in file).""" + n = len(df) + step = df["step"].to_numpy(np.int64) + amount = df["amount"].to_numpy(np.float64) + is_transfer = (df["type"] == "TRANSFER").to_numpy() + codes, _ = pd.factorize(pd.concat([df["nameDest"], df["nameOrig"]], ignore_index=True)) + dest, orig = codes[:n], codes[n:] + out = df.copy() + + # --- receiver: rows sorted by (receiver, time) + order = np.lexsort((np.arange(n), dest)) + sd, st, sa, stf = dest[order], step[order], amount[order], is_transfer[order] + idx = np.arange(n) + new = np.r_[True, sd[1:] != sd[:-1]] + start = np.maximum.accumulate(np.where(new, idx, 0)) + prior = idx - start + pa, pt = _prefix(sa), _prefix(stf.astype(np.float64)) + key = sd.astype(np.int64) * 1000 + st # steps are below 1000 + first_in_window = np.searchsorted(key, key - WINDOW, side="right") + prev_step = np.where(new, np.nan, np.r_[np.nan, st[:-1]]) + prior_value = pa[idx] - pa[start] + feats = { + "r_prior_in": prior, + "r_prior_in_24h": idx - first_in_window, + "r_prior_in_value_24h": pa[idx] - pa[first_in_window], + "r_hours_since_last_in": st - prev_step, + "r_hours_since_first_seen": st - st[start], + "r_prior_in_transfers": pt[idx] - pt[start], + "amount_vs_r_mean_in": np.where( + prior > 0, sa / np.maximum(prior_value / np.maximum(prior, 1), 1e-9), np.nan + ), + } + for name, values in feats.items(): + col = np.empty(n, dtype=np.float64) + col[order] = values + out[name] = col + + # --- sender: earlier payments sent, and the last money it received before this row + out["s_prior_out"] = out.groupby(orig).cumcount().to_numpy(np.float64) + receipts = pd.DataFrame({"account": dest, "t": idx, "r_step": step, "r_amount": amount}) + queries = pd.DataFrame({"account": orig, "t": idx}) + last = pd.merge_asof( + queries, receipts, on="t", by="account", allow_exact_matches=False, direction="backward" + ) + out["s_hours_since_received"] = step - last["r_step"].to_numpy(np.float64) + out["amount_vs_s_received"] = amount / last["r_amount"].to_numpy(np.float64) + + out["is_transfer"] = is_transfer.astype(np.float64) + out["hour_of_day"] = (step % 24).astype(np.float64) + out["amount_vs_balance"] = np.where( + df["oldbalanceOrg"] > 0, amount / df["oldbalanceOrg"].clip(lower=1e-9), np.nan + ) + return out + + +def split(df: pd.DataFrame) -> dict[str, pd.DataFrame]: + scored = df[df["type"].isin(SCORED_TYPES)] + return { + "train": scored[scored["step"] <= TRAIN_LAST_STEP], + "val": scored[(scored["step"] > TRAIN_LAST_STEP) & (scored["step"] <= VAL_LAST_STEP)], + "test": scored[scored["step"] > VAL_LAST_STEP], + } + + +def evaluate(df: pd.DataFrame, replicates: int) -> dict: + folds = split(df) + y = {k: v["isFraud"].to_numpy() for k, v in folds.items()} + behaviour, with_balances = list(BEHAVIOUR_FEATURES), [*BEHAVIOUR_FEATURES, *BALANCE_FEATURES] + scores: dict[str, tuple[np.ndarray, np.ndarray]] = {} + + # Simple baselines first. + scores["paysim_flag_rule"] = tuple( # PaySim's own rule: TRANSFER over 200,000 + folds[k]["isFlaggedFraud"].to_numpy(np.float64) for k in ("val", "test") + ) + scores["amount_only"] = tuple(folds[k]["amount"].to_numpy() for k in ("val", "test")) + logistic = fit_logistic(folds["train"][behaviour], y["train"]) + scores["logistic_regression"] = tuple( + logistic.predict_proba(finite(folds[k][behaviour]))[:, 1] for k in ("val", "test") + ) + forest = fit_isolation_forest(folds["train"][behaviour]) + scores["isolation_forest"] = tuple( + anomaly_score(forest, folds[k][behaviour]) for k in ("val", "test") + ) + + # Hour of day is partly a simulator artefact too: PaySim spreads fraud evenly over + # the day while legitimate traffic follows a daily cycle. Without it, what is left + # is receiver-centric, velocity and amount signal. + no_hour = [c for c in behaviour if c != "hour_of_day"] + boosters = {} + for name, cols in ( + ("lightgbm", behaviour), + ("lightgbm_without_hour", no_hour), + ("lightgbm_with_balances", with_balances), + ): + booster = fit_lightgbm(folds["train"][cols], y["train"], folds["val"][cols], y["val"]) + boosters[name] = booster + scores[name] = tuple( + booster.predict(folds[k][cols].to_numpy(np.float64)) for k in ("val", "test") + ) + + results = { + name: judge(y["val"], s_val, y["test"], s_test, FPRS) + for name, (s_val, s_test) in scores.items() + } + steps = folds["test"]["step"].to_numpy() + for name in boosters: + s_val, s_test = scores[name] + results[name]["bootstrap_by_hour"] = bootstrap_ci( + y["val"], s_val, y["test"], s_test, steps, CI_FPR, replicates + ) + results[name]["trees"] = boosters[name].num_trees() + results[name]["top_features"] = top_gain(boosters[name]) + + return { + "dataset": "PaySim (Lopez-Rojas et al., 2016)", + "source": download.SOURCES["paysim"].url, + "sha256": download.SOURCES["paysim"].sha256, + "rows_total": int(len(df)), + "scored_types": list(SCORED_TYPES), + "split_by_hour": { + "train": f"1-{TRAIN_LAST_STEP}", + "val": f"{TRAIN_LAST_STEP + 1}-{VAL_LAST_STEP}", + "test": f"{VAL_LAST_STEP + 1}-{int(df['step'].max())}", + }, + "folds": { + k: { + "rows": int(len(v)), + "fraud": int(v["isFraud"].sum()), + "base_rate": round(float(v["isFraud"].mean()), 5), + } + for k, v in folds.items() + }, + "features": {"behaviour": behaviour, "balances_added": list(BALANCE_FEATURES)}, + "thresholds": "fixed on val at each target FPR, applied unchanged to test", + "models": results, + } + + +def run(path: Path | None = None, out_dir: Path | None = None, replicates: int = 500) -> dict: + t0 = time.perf_counter() + df = add_features(load(path or download.fetch("paysim"))) + report = evaluate(df, replicates) + report["seconds"] = round(time.perf_counter() - t0, 1) + out = out_dir or settings.artifacts_dir / "external" + out.mkdir(parents=True, exist_ok=True) + (out / "paysim.json").write_text(json.dumps(report, indent=2)) + return report + + +def print_summary(report: dict) -> None: + print(f"\n{report['dataset']}: folds {json.dumps(report['folds'])}") + rows = { + name: { + "pr_auc": m["pr_auc"], + "roc_auc": m["roc_auc"], + **{f"recall@{f:g}": m[f"at_fpr_{f:g}"]["recall"] for f in FPRS}, + **{f"testfpr@{f:g}": m[f"at_fpr_{f:g}"]["fpr_on_test"] for f in FPRS}, + } + for name, m in report["models"].items() + } + print(pd.DataFrame(rows).T.to_string()) + for name in ("lightgbm", "lightgbm_without_hour", "lightgbm_with_balances"): + print(name, json.dumps(report["models"][name]["bootstrap_by_hour"])) + print(" top features:", report["models"][name]["top_features"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description="FraudLens approach on PaySim") + parser.add_argument("--replicates", type=int, default=500) + print_summary(run(replicates=parser.parse_args().replicates)) + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/intel/analyze.py b/backend/src/fraudlens/intel/analyze.py index 641a4b7..6d5d6d3 100644 --- a/backend/src/fraudlens/intel/analyze.py +++ b/backend/src/fraudlens/intel/analyze.py @@ -4,10 +4,14 @@ reads the words (`text.py`), the link check reads where the links go (`links.py`). The text is read and discarded. It is not stored, not logged and, like every other piece of customer free text in the platform, never treated as an instruction. + +The reasons point into the message itself (`explain.py`): each signal carries the +phrases that matched it, and `highlights` names the words that raised the score most. """ from __future__ import annotations +from .explain import cue_phrases, highlights from .links import check_links, worst from .taxonomy import Taxonomy from .text import TextModel @@ -25,14 +29,19 @@ def check_message(text: str, model: TextModel | None, taxonomy: Taxonomy) -> dict: """`model` is None when the classifier has not been trained; links are still checked.""" links = check_links(text, taxonomy.brands) - level, risk, named, signals = worst(links), None, {}, [] + level, risk, named, signals, words = worst(links), None, {}, [], [] if model is not None: scores = model.score([text])[0] risk = round(float(scores[0]), 4) text_level = model.level(risk) if text_level != "none": - signals = [{"id": cue.id, "label": cue.label} for cue in model.signals(text)] + # Each cue with the stretch of the customer's own text that showed it. + signals = [ + {"id": cue.id, "label": cue.label, "phrases": phrases} + for cue, phrases in cue_phrases(text) + ] + words = highlights(text, model.pipeline) for cid, p in zip(model.categories, scores[1:], strict=True): if p >= model.thresholds["category"]: named[cid] = {"probability": round(float(p), 4), "source": "model"} @@ -61,6 +70,7 @@ def check_message(text: str, model: TextModel | None, taxonomy: Taxonomy) -> dic "risk": risk, "categories": categories, "signals": signals, + "highlights": words, "links": links, "advice": advice, "model_version": model.version if model is not None else None, diff --git a/backend/src/fraudlens/intel/compare.py b/backend/src/fraudlens/intel/compare.py new file mode 100644 index 0000000..2c818fe --- /dev/null +++ b/backend/src/fraudlens/intel/compare.py @@ -0,0 +1,290 @@ +"""Which model reads scam messages best: the evidence behind the production choice. + + python -m fraudlens.intel.compare # or: make intel-compare + +Trains each candidate for the scam-or-not output on the same rows, sets each one's +caution threshold the same way (at most 3% of each source's harmless validation +messages flagged), +and measures it on three sets it never trained on: + +- `test`: corpus scripts it trained on, in wordings it never saw. +- `unseen`: corpus scripts (one per typology, and harmless topics) kept out of training. +- `external`: the public corpus's own test split (`external.py`), written by people + who are not the author of this one. + +Candidates are the cues alone, word TF-IDF, character n-gram TF-IDF, character +n-grams with cues (the v1 recipe), and a small multilingual sentence encoder +(paraphrase-multilingual-MiniLM-L12-v2) with logistic regression on top, each +trained on the corpus alone and on the corpus plus the public training split. The +encoder runs only when the optional `transformer` extra is installed +(`uv sync --extra transformer`); without it, it is reported as skipped. + +Writes `compare.json` next to the model. It does not change the served model. +""" + +from __future__ import annotations + +import argparse +import json +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import f1_score, roc_auc_score +from sklearn.pipeline import FeatureUnion, Pipeline + +from ..config import Settings +from . import external +from .corpus import TYPOLOGIES, generate +from .cues import CueFeatures +from .text import intel_dir, normalise + +COMPARE_FILE = "compare.json" +CAUTION_FPR = 0.03 +ENCODER = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" +LATENCY_MESSAGES = 200 + + +@dataclass(frozen=True) +class Row: + text: str + scam: bool + lang: str + typology: str | None # corpus scams only + source: str # corpus | external + + +def rows(seed: int, per_template: int, settings: Settings) -> dict[str, list[Row]]: + """Every split, corpus and public. Public rows are absent when they cannot be fetched.""" + out: dict[str, list[Row]] = {} + for s in generate(seed=seed, per_template=per_template): + row = Row(s.text, bool(s.labels), s.lang, s.typology, "corpus") + out.setdefault(s.split, []).append(row) + public = external.ensure(settings) + for split, items in (public or {}).items(): + out[f"external_{split}"] = [Row(e.text, e.scam, e.lang, None, "external") for e in items] + return out + + +# ------------------------------------------------------------------ candidates + + +def _lr(seed: int) -> LogisticRegression: + return LogisticRegression(C=2.0, max_iter=3000, class_weight="balanced", random_state=seed) + + +def _chars() -> TfidfVectorizer: + return TfidfVectorizer( + analyzer="char_wb", ngram_range=(2, 5), min_df=2, sublinear_tf=True, + max_features=60_000, lowercase=False, + ) # fmt: skip + + +def _sparse(features, seed: int) -> Callable: + def fit(texts: list[str], y: np.ndarray) -> Callable[[list[str]], np.ndarray]: + pipe = Pipeline([("features", features()), ("model", _lr(seed))]) + pipe.fit([normalise(t) for t in texts], y) + return lambda batch: pipe.predict_proba([normalise(t) for t in batch])[:, 1] + + return fit + + +def _encoder_fit(seed: int): + try: + from sentence_transformers import SentenceTransformer + except ImportError: + return None + encoder = SentenceTransformer(ENCODER, device="cpu") + + def encode(batch: list[str]) -> np.ndarray: + return encoder.encode( + [t[:2000] for t in batch], batch_size=64, normalize_embeddings=True, + show_progress_bar=False, + ) # fmt: skip + + cache: dict[str, np.ndarray] = {} + + def embed(batch: list[str]) -> np.ndarray: + todo = [t for t in dict.fromkeys(batch) if t not in cache] + if todo: + cache.update(zip(todo, encode(todo), strict=True)) + return np.stack([cache[t] for t in batch]) + + def fit(texts: list[str], y: np.ndarray): + model = _lr(seed).fit(embed(texts), y) + scorer = lambda batch: model.predict_proba(embed(batch))[:, 1] # noqa: E731 + # Latency is measured without the cache: a new message is encoded from scratch. + scorer.uncached = lambda batch: model.predict_proba(encode(batch))[:, 1] + return scorer + + return fit + + +def candidates(seed: int) -> dict[str, Callable | None]: + def cues(): + return CueFeatures() + + def words(): + return TfidfVectorizer(ngram_range=(1, 2), min_df=2, sublinear_tf=True, lowercase=False) + + def chars_cues(): + return FeatureUnion( + [("chars", _chars()), ("cues", CueFeatures())], + transformer_weights={"chars": 1.0, "cues": 0.5}, + ) + + return { + "cues_only": _sparse(cues, seed), + "word_tfidf": _sparse(words, seed), + "char_tfidf": _sparse(_chars, seed), + "char_cues": _sparse(chars_cues, seed), + "minilm_lr": _encoder_fit(seed), + } + + +# --------------------------------------------------------------------- metrics + + +def _macro_f1(y: np.ndarray, flagged: np.ndarray) -> float | None: + if len(y) == 0 or len(set(y)) < 2: + return None + return round(float(f1_score(y, flagged, average="macro")), 4) + + +def measure(items: list[Row], scores: np.ndarray, threshold: float) -> dict: + y = np.array([r.scam for r in items], dtype=int) + flagged = (scores >= threshold).astype(int) + langs = np.array([r.lang for r in items]) + out = { + "messages": len(items), + "scam": int(y.sum()), + "auc": round(float(roc_auc_score(y, scores)), 4), + "macro_f1": _macro_f1(y, flagged), + "recall": round(float(flagged[y == 1].mean()), 4), + "false_positive_rate": round(float(flagged[y == 0].mean()), 4), + "languages": { + lang: _macro_f1(y[langs == lang], flagged[langs == lang]) for lang in sorted(set(langs)) + }, + } + typ = np.array([r.typology or "" for r in items]) + if typ.any(): + # Each typology's scams against every harmless message of the same split. + out["typologies"] = {} + for t in TYPOLOGIES: + keep = (typ == t) | (y == 0) + if (typ == t).any(): + out["typologies"][t] = { + "scam": int((typ == t).sum()), + "macro_f1": _macro_f1(y[keep], flagged[keep]), + "recall": round(float(flagged[typ == t].mean()), 4), + } + return out + + +def caution_threshold(val: list[Row], scores: np.ndarray, fpr: float = CAUTION_FPR) -> float: + """The lowest score that flags at most `fpr` of the harmless validation messages + of each source. The public harmless messages are mostly easy (promotions, chat); + pooled with ours they would let the threshold drift onto our hard ones.""" + out = 0.0 + for source in {r.source for r in val}: + benign = scores[[r.source == source and not r.scam for r in val]] + if len(benign): + out = max(out, float(np.quantile(benign, 1 - fpr, method="higher")) + 1e-6) + return out + + +def latency(score: Callable, texts: list[str]) -> dict: + """Milliseconds to score one message at a time on this machine's CPU.""" + score(texts[:5]) # warm up + times = [] + for t in texts: + start = time.perf_counter() + score([t]) + times.append((time.perf_counter() - start) * 1000) + return { + "p50_ms": round(float(np.percentile(times, 50)), 2), + "p95_ms": round(float(np.percentile(times, 95)), 2), + } + + +# ------------------------------------------------------------------------ main + + +def compare(directory: Path, settings: Settings, seed: int = 7, per_template: int = 30) -> dict: + data = rows(seed, per_template, settings) + has_public = "external_train" in data + val = data["val"] + data.get("external_val", []) + evals = {"test": data["test"], "unseen": data["unseen"]} + if has_public: + evals["external"] = data["external_test"] + train_sets = {"corpus": data["train"]} + if has_public: + train_sets["corpus+external"] = data["train"] + data["external_train"] + + rng = np.random.default_rng(seed) + probe = [r.text for r in rng.permutation(np.array(val, dtype=object))[:LATENCY_MESSAGES]] + + results: dict = {} + for name, fit in candidates(seed).items(): + if fit is None: + results[name] = {"skipped": "install the optional extra: uv sync --extra transformer"} + continue + for train_name, train_rows in train_sets.items(): + key = f"{name}+ext" if train_name != "corpus" else name + start = time.perf_counter() + score = fit([r.text for r in train_rows], np.array([r.scam for r in train_rows])) + fit_s = time.perf_counter() - start + val_scores = score([r.text for r in val]) + threshold = caution_threshold(val, val_scores) + entry = { + "trained_on": train_name, + "train_messages": len(train_rows), + "fit_seconds": round(fit_s, 1), + "caution_threshold": round(threshold, 6), + "latency": latency(getattr(score, "uncached", score), probe), + } + for split, items in evals.items(): + entry[split] = measure(items, score([r.text for r in items]), threshold) + results[key] = entry + print(_line(key, entry), flush=True) + + out = { + "seed": seed, + "threshold": { + "caution_fpr": CAUTION_FPR, + "set_on": "harmless val messages of each source; the stricter of the two", + }, + "encoder": ENCODER, + "public_corpus": external.provenance() if has_public else None, + "sets": {k: len(v) for k, v in data.items()}, + "candidates": results, + } + directory.mkdir(parents=True, exist_ok=True) + (directory / COMPARE_FILE).write_text(json.dumps(out, indent=2)) + return out + + +def _line(key: str, e: dict) -> str: + parts = [f"{key:16}"] + for split in ("test", "unseen", "external"): + if split in e: + parts.append(f"{split} auc {e[split]['auc']:.3f} f1 {e[split]['macro_f1']:.3f}") + parts.append(f"p50 {e['latency']['p50_ms']}ms") + return " ".join(parts) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare scam-message models") + parser.add_argument("--seed", type=int, default=7) + args = parser.parse_args() + settings = Settings() + compare(intel_dir(settings), settings, seed=args.seed) + print(f"written to {intel_dir(settings) / COMPARE_FILE}") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/intel/corpus.py b/backend/src/fraudlens/intel/corpus.py index 15f6910..66ef43f 100644 --- a/backend/src/fraudlens/intel/corpus.py +++ b/backend/src/fraudlens/intel/corpus.py @@ -29,7 +29,18 @@ from .taxonomy import load_taxonomy CORPUS_PATH = Path(__file__).parent / "corpus.yaml" -LANGUAGES = {"bn": "Bangla", "bl": "Bangla in Latin letters", "en": "English"} +LANGUAGES = { + "bn": "Bangla", + "bl": "Bangla in Latin letters", + "en": "English", + "mx": "Bangla and English mixed", +} +# The scam typologies the corpus is organised around (the scripts customers meet +# most often), so results can be read per typology. `other` covers the rest. +TYPOLOGIES = ( + "fake_agent_helpline", "prize_lottery", "wrong_number", "job_investment", + "govt_aid", "otp_pin_phishing", "other", +) # fmt: skip _BN_DIGITS = str.maketrans("0123456789", "০১২৩৪৫৬৭৮৯") @@ -57,11 +68,13 @@ "bn": ["আসসালামু আলাইকুম। ", "ভাই, ", "প্রিয় গ্রাহক, ", "স্যার, ", "আপা, "], "bl": ["Assalamu alaikum. ", "vai ", "Sir ", "apu ", "hello ", "bhai, "], "en": ["Hello. ", "Dear customer, ", "Hi, ", "Sir, ", "Good day. "], + "mx": ["Hello ভাই, ", "Dear customer, ", "Sir, ", "আপু, ", "Hi, "], } _CLOSERS = { "bn": [" ধন্যবাদ।", " ভালো থাকবেন।", ""], "bl": [" dhonnobad", " thanks", " pls", ""], "en": [" Thanks.", " Thank you.", " Regards.", ""], + "mx": [" Thanks.", " ধন্যবাদ।", " please", ""], } # How the same Bangla word is spelled by different people writing in Latin letters. @@ -75,7 +88,6 @@ "theke": ["thk", "thake", "theike"], "hoye": ["hoe", "hoia"], "jabe": ["jbe", "jaabe"], - "pathan": ["pathiye din", "send korun", "pathay den"], "vul": ["bhul", "bul"], "kore": ["kre", "koira"], "number": ["nambar", "no", "nmbr"], @@ -91,6 +103,13 @@ "code": ["kod", "cod"], "ferot": ["ferat", "back"], "geche": ["gese", "gechhe"], + "ekhon": ["akhon", "ekhn"], + "ekhoni": ["akhoni", "ekhuni"], + "taratari": ["tartari", "taratari"], + "pathan": ["pathaan", "pathiye den", "send koren"], + "bolen": ["bolun", "bolen na"], + "korsi": ["korchi", "korechi"], + "gese": ["geche", "gechhe"], } _SLOT = re.compile(r"\{(\w+)\}") @@ -103,6 +122,7 @@ class Family: templates: tuple[tuple[str, str], ...] # (language, text) hard: bool = False held_out: bool = False + typology: str | None = None # scams only: one of TYPOLOGIES @dataclass(frozen=True) @@ -113,6 +133,7 @@ class Sample: lang: str split: str # train | val | test | unseen hard: bool + typology: str | None = None @cache @@ -127,9 +148,12 @@ def load_families(path: Path = CORPUS_PATH) -> tuple[Family, ...]: templates=tuple((lang, text) for lang, text in item["templates"]), hard=bool(item.get("hard", False)), held_out=bool(item.get("held_out", False)), + typology=item.get("typology", "other") if item["labels"] else None, ) if set(family.labels) - known: raise ValueError(f"family {family.id} names an unknown category") + if family.typology is not None and family.typology not in TYPOLOGIES: + raise ValueError(f"family {family.id} names an unknown typology") if any(lang not in LANGUAGES for lang, _ in family.templates): raise ValueError(f"family {family.id} has a template in an unknown language") families.append(family) @@ -254,5 +278,9 @@ def generate(seed: int = 7, per_template: int = 30) -> list[Sample]: if text in seen: # a template with no slots renders few distinct texts continue seen.add(text) - samples.append(Sample(text, family.labels, family.id, lang, split, family.hard)) + samples.append( + Sample( + text, family.labels, family.id, lang, split, family.hard, family.typology + ) + ) return samples diff --git a/backend/src/fraudlens/intel/corpus.yaml b/backend/src/fraudlens/intel/corpus.yaml index 4dc5362..6b8e2fb 100644 --- a/backend/src/fraudlens/intel/corpus.yaml +++ b/backend/src/fraudlens/intel/corpus.yaml @@ -2,20 +2,25 @@ # # Every line is written for this project. None is a real message from a real # person: they are modelled on the scripts fraud teams and news reports describe -# in Bangladesh, in the three ways people actually write (bn: Bangla script, -# bl: Bangla in Latin letters, en: English). +# in Bangladesh, in the ways people actually write (bn: Bangla script, +# bl: Bangla in Latin letters, en: English, mx: Bangla script mixed with English +# words). Version 2 added the `mx` templates, the `typology` tags, three harmless +# look-alike families and six held-out scripts, one per typology. # # A family is one script. `labels` are taxonomy category ids; a family without # labels is ordinary, harmless text. `hard: true` marks harmless text that # shares words with scams (a real OTP notice, a parent asked for money). # `held_out: true` families are never trained on: they measure what the model -# does with a script it has not met. +# does with a script it has not met. `typology` groups scam families into the +# scripts customers meet most (fake agent or helpline, prize or lottery, wrong +# number, job or investment, government aid, OTP or PIN phishing); default `other`. # # Slots in braces are filled by fraudlens.intel.corpus. families: # ------------------------------------------------------------------ scams - id: otp_call_block + typology: otp_pin_phishing labels: [social_engineering, impersonation, account_takeover] templates: - [bl, "{brand} office theke bolchi. apnar account e somossa hoyeche, block hoye jabe. apnar phone e ekta OTP geche, seta bolun"] @@ -25,8 +30,12 @@ families: - [en, "This is {brand} customer service. Your account will be blocked today. Tell me the verification code we just sent to your phone to keep it active."] - [en, "Dear customer, your {brand} wallet is under review. Share the OTP you received within 5 minutes or the account will be closed permanently."] - [bl, "apnar {brand} account ta block hoye gese, thik korte hole ekhon je code jabe oita amake bolun, na hole taka sob kete jabe"] + - [mx, "{brand} head office থেকে বলছি। আপনার account temporary block করা হয়েছে, unblock করতে phone এ আসা OTP টা বলুন।"] + - [bl, "assalamualaikum, {brand} customer service theke call korsi, apnar acc te problem, ekhn 1ta code jabe, oita amake bolle sob thik"] + - [en, "Your {brand} account has been flagged. Our officer is on the line, read out the 6-digit OTP to stop the closure."] - id: pin_verify + typology: otp_pin_phishing labels: [social_engineering, impersonation, account_takeover] templates: - [bl, "{brand} theke bolchi, apnar account verify korar jonno PIN number ta bolun, na hole account bondho hoye jabe"] @@ -36,8 +45,11 @@ families: - [bn, "{brand_bn} কাস্টমার কেয়ার থেকে বলছি। আপনার অ্যাকাউন্ট সচল রাখতে পিন নম্বরের সাথে {otp} যোগ করে যে সংখ্যা হয় সেটি বলুন।"] - [en, "For security verification please type your {brand} PIN followed by the code {otp} and send it to this number."] - [bl, "apnar pin er sathe {otp} jog kore je sonkha hoy seta bolun, tahole apnar account abar chalu hoye jabe"] + - [mx, "আপনার {brand} account verify করতে PIN number টা confirm করুন, না হলে account suspend হবে।"] + - [bl, "apner wallet er pin ta bolen sir, system update hocche, pin na dile taka tulte parben na"] - id: refund_code + typology: otp_pin_phishing labels: [social_engineering, impersonation] templates: - [bl, "apnar {brand} account e {amt} taka refund asbe, refund pete phone e asa code ta amake din"] @@ -46,6 +58,7 @@ families: - [bl, "sir apnar cashback {amt} tk pending ase, ami pathiye dicchi, sudhu sms er code ta bolen"] - [bn, "{brand_bn} থেকে আপনার ক্যাশব্যাকের {amt_bn} টাকা আটকে আছে। ছাড়াতে এখনই কোডটি বলুন, নইলে বাতিল হয়ে যাবে।"] - [en, "Your cashback of BDT {amt} is on hold. Our officer will call you, give him the 6 digit code to release it."] + - [mx, "Sir আপনার {amt_bn} টাকা refund pending আছে, process করতে phone এ যাওয়া code টা দিন।"] - id: relative_emergency labels: [social_engineering] @@ -56,6 +69,7 @@ families: - [bl, "vai ami accident korsi hospital e, kauke bolo na, {num} ei number e {amt} tk pathao joldi"] - [bn, "মামা আমি পুলিশের কাছে আটকা পড়েছি, কাউকে বলবেন না। {amt_bn} টাকা এই নম্বরে এখনই পাঠান, নইলে বড় বিপদ হবে।"] - [en, "Mom my number changed. I need {amt} urgently for a hospital bill, send it to {num}, this is my friend's {brand}. Please hurry."] + - [mx, "Ammu আমি {name_bn}, phone নষ্ট, এটা friend এর number। hospital এ আছি, urgent {amt_bn} টাকা পাঠাও {num_bn}।"] - id: romance_gift labels: [social_engineering] @@ -66,8 +80,10 @@ families: - [en, "Dear, I am a doctor working abroad. I want to visit you but my account is frozen, lend me {amt} taka on {brand} and I will return double."] - [bl, "ami tomake onek valobashi, ekta problem e porsi, {amt} tk dhar dao, kal e ferot dibo, {num} e pathao"] - [bn, "তোমাকে ছাড়া আর কাউকে বলতে পারছি না। আমার মায়ের অপারেশন, {amt_bn} টাকা দরকার, বিকাশ করে দাও, আমি এসে ফেরত দেব।"] + - [mx, "Darling তোমার জন্য gift পাঠিয়েছি, customs এ আটকে আছে, release করতে {amt_bn} টাকা পাঠাও {num_bn}।"] - id: fake_helpline + typology: fake_agent_helpline labels: [impersonation, social_engineering] templates: - [bn, "{brand_bn} কাস্টমার কেয়ার হেল্পলাইন {num_bn}। যেকোনো সমস্যায় ২৪ ঘণ্টা এই নম্বরে কল করুন অথবা ইনবক্স করুন।"] @@ -76,8 +92,12 @@ families: - [bn, "আপনার অভিযোগটি আমরা দেখেছি। দ্রুত সমাধানের জন্য আমাদের {brand_bn} প্রতিনিধির সাথে {num_bn} নম্বরে যোগাযোগ করুন।"] - [en, "We saw your complaint about {brand}. Please inbox your wallet number and contact our help desk officer on {num}."] - [bl, "apnar complain ta peyechi. {brand} support officer er sathe kotha bolte {num} e call din, inbox e wallet number din"] + - [mx, "{brand} Customer Care Official Page: আপনার complain টা পেয়েছি, solution এর জন্য {num_bn} এ WhatsApp করুন।"] + - [bl, "taka vul number e gele ferot ante {brand} er helpline e call din {num}, 24 ghonta khola"] + - [en, "Lost money in a wrong transfer? {brand} recovery desk can reverse it. Call our officer at {num}, available 24/7."] - id: fake_agent + typology: fake_agent_helpline labels: [impersonation, social_engineering, fake_payment] templates: - [bl, "vai ami agent, apni ektu age amar dokan theke cash in korsilen. vul kore {amt} taka beshi chole geche, taka ta ferot pathan"] @@ -86,8 +106,11 @@ families: - [bl, "ami {brand} agent bolchi, apnar cash in ta duibar hoye gese, {amt} taka back koren, na hole apnar account block kore dibo"] - [bn, "ভাই আমি এজেন্ট, আপনার নম্বরে ভুলে {amt_bn} টাকা ঢুকে গেছে। গরিব মানুষ, টাকাটা এই নম্বরে ফেরত দিন {num_bn}।"] - [en, "Agent here. Your cash in is stuck in the system. To release it I need the code that just came to your phone."] + - [mx, "আমি আপনার area র {brand} agent, cash in এর সময় ভুলে {amt_bn} টাকা বেশি গেছে, please এই number এ back করেন {num_bn}।"] + - [bl, "ami agent dokan theke bolchi, apnar cash in ta 2 bar hoye gese, ekta ferot pathan {num} e, na hole complain korbo"] - id: govt_stipend + typology: govt_aid labels: [impersonation, scam_campaign, social_engineering] templates: - [bn, "শিক্ষা বোর্ড থেকে বলছি। আপনার সন্তানের উপবৃত্তির {amt_bn} টাকা এসেছে। টাকা পেতে আপনার {brand_bn} নম্বর ও পিন বলুন।"] @@ -97,8 +120,13 @@ families: - [bl, "apnar boyosko vatar taka asche, receive korte {fee} taka charge ei number e pathan {num}"] - [bn, "সরকারি ত্রাণ তহবিল থেকে আপনি {amt_bn} টাকা অনুদান পেয়েছেন। টাকা তুলতে {fee_bn} টাকা ফি এই নম্বরে পাঠান।"] - [en, "Govt allowance department: Tk {amt} has been approved for you. Share your wallet PIN with our officer to receive the money."] + - [mx, "শিক্ষা বোর্ড থেকে stipend এর {amt_bn} টাকা আসবে, confirm করতে আপনার {brand} PIN টা বলুন।"] + - [bl, "protibondhi vatar taka release hobe, apnar {brand} number ar code ta bolen, ami office theke entry dicchi"] + - [en, "Ministry of Social Welfare: your widow allowance of Tk {amt} is pending. Pay a verification fee of Tk {fee} to {num} to release it."] + - [bn, "প্রাথমিক শিক্ষা অধিদপ্তর থেকে বলছি, আপনার সন্তানের উপবৃত্তির টাকা আটকে আছে। ছাড়াতে {fee_bn} টাকা ফি পাঠান {num_bn} নম্বরে।"] - id: wrong_send + typology: wrong_number labels: [fake_payment, social_engineering] templates: - [bl, "vai vul kore apnar number e {amt} taka chole geche, doya kore ferot din ei number e {num}"] @@ -108,6 +136,9 @@ families: - [bl, "apnar {brand} e {amt} tk vul kore pathiye disi, msg dekhen, taka ta back koren pls, na hole thanay jabo"] - [bn, "Cash In Tk {amt} সফল হয়েছে। TrxID {trx}। ভাই এটা ভুলে গেছে, এখনই এই নম্বরে ফেরত দিন, নইলে মামলা করব।"] - [bl, "apu amar baba r oshudh er taka vul e apnar nambar e chole gese {amt} taka, ektu ferot den na pls"] + - [mx, "Bhai ভুল করে আপনার number এ {amt_bn} টাকা send হয়ে গেছে, please এই number এ ফেরত দেন {num_bn}।"] + - [bl, "vaia ami vul kre apnar akaunt e {amt} tk send money kore felsi, amar meyer exam fee, plz back den"] + - [en, "Sir, my {amt} taka went to your {brand} by mistake while paying the shop. Kindly return it, it is all I have."] - id: screenshot_pressure labels: [fake_payment, merchant_fraud] @@ -118,8 +149,10 @@ families: - [bl, "taka pathano hoye gese, sms asbe pore, network slow. apni mal ta rider er kase diye den taratari"] - [en, "I already paid {amt} by {brand}, the SMS is delayed. Don't check the app, just trust the screenshot and release the order."] - [bn, "টাকা পাঠিয়ে দিয়েছি, মেসেজ আসতে দেরি হবে। অ্যাপে চেক করার দরকার নেই, স্ক্রিনশটই প্রমাণ, মাল ছেড়ে দিন।"] + - [mx, "Payment করে দিয়েছি, screenshot দেখেন। Server slow তাই balance এ পরে আসবে, product টা এখনই পাঠান।"] - id: fake_reversal + typology: wrong_number labels: [fake_payment, social_engineering] templates: - [en, "Your transaction {trx} of Tk {amt} has failed and the amount is on hold. To reverse it, send the same amount again to {num}."] @@ -128,8 +161,10 @@ families: - [en, "Refund notice: Tk {amt} will be returned to your wallet after you pay a reversal charge of Tk {fee} to {num}."] - [bl, "apnar taka vul account e chole gese, reverse korte {fee} tk charge lagbe, ei number e pathan"] - [bn, "আপনার টাকা ভুল নম্বরে চলে গেছে বলে অভিযোগ পেয়েছি। ফেরত আনতে {fee_bn} টাকা সার্ভিস চার্জ পাঠান।"] + - [mx, "আপনার transaction টা fail হয়েছে, refund পেতে আবার {amt_bn} টাকা এই number এ send করুন {num_bn}।"] - id: sim_upgrade + typology: otp_pin_phishing labels: [account_takeover, social_engineering, impersonation] templates: - [bl, "apnar SIM ta 4G te upgrade korte hobe, na hole bondho hoye jabe. phone e je code jabe seta bolun"] @@ -138,8 +173,10 @@ families: - [bl, "apnar sim re-registration korte hobe, NID number ar sms e asa code ta den, na hole 2 ghontar moddhe sim off"] - [bn, "আপনার সিমটি ফোরজি থেকে ফাইভজি করা হচ্ছে। কিছুক্ষণ ফোন বন্ধ রাখুন এবং তার আগে এসএমএসে আসা কোডটি জানান।"] - [en, "We are replacing your SIM with a new one for free. Keep your phone switched off for two hours and tell us the code you received."] + - [mx, "আপনার SIM এর biometric verification বাকি, SIM block হওয়ার আগে SMS এ আসা code টা বলুন।"] - id: code_by_mistake + typology: otp_pin_phishing labels: [account_takeover, social_engineering] templates: - [bl, "vai vul kore amar ekta code apnar number e chole gese, 6 digit er, ektu bolen na pls"] @@ -148,6 +185,8 @@ families: - [bl, "apu amar {brand} er code ta apnar phone e gese vul e, forward kore den pls khub dorkar"] - [en, "A login code was sent to your phone in error. Please forward that message to this number so I can open my account."] - [bn, "আমার নতুন ফোনে অ্যাকাউন্ট চালু করছি, কোডটা ভুলে আপনার কাছে গেছে। মেসেজটা আমাকে ফরওয়ার্ড করে দিন।"] + - [mx, "Bhai ভুল করে আমার verification code আপনার number এ চলে গেছে, please code টা forward করে দেন।"] + - [bl, "sorry vai, amr {brand} er code ta apnar nmbr e chole gese, 6 ta digit just bolen"] - id: advance_payment_shop labels: [merchant_fraud] @@ -158,6 +197,7 @@ families: - [bl, "stock sesh hoye jacche, ekhoni {amt} taka advance send money koren {num} e, na hole order cancel"] - [bn, "{item_bn} বুকিং দিতে {amt_bn} টাকা অগ্রিম সেন্ড মানি করুন। টাকা না পাঠালে অর্ডার নেওয়া হবে না, স্টক সীমিত।"] - [en, "Limited stock! Pay Tk {amt} booking money now by send money to {num}. Delivery after full payment only."] + - [mx, "Order confirm করতে full payment advance করতে হবে, {brand} personal number এ send money করুন। Cash on delivery নেই।"] - id: courier_fee_first labels: [merchant_fraud] @@ -168,6 +208,7 @@ families: - [bl, "bike ta dekhte chaile age {fee} tk booking den, onek customer ase, na dile onno jon ke diye dibo"] - [en, "I am an army officer being transferred, selling my {item} cheap. Send Tk {fee} transport cost first and it will reach you tomorrow."] - [bn, "আমি সেনাবাহিনীতে চাকরি করি, বদলি হয়ে যাচ্ছি তাই {item_bn} কম দামে দিচ্ছি। পরিবহন খরচ {fee_bn} টাকা আগে পাঠান।"] + - [mx, "{item_bn} টা courier এ পাঠাবো, আগে delivery charge {fee_bn} টাকা send করুন, আমি Dhaka র বাইরে।"] - id: deep_discount labels: [merchant_fraud, scam_campaign] @@ -178,8 +219,10 @@ families: - [en, "Flash sale for the first 100 buyers only. {item} for Tk {amt}. Advance payment required to {num}."] - [bn, "প্রথম ১০০ জনের জন্য {item_bn} {pct}% ছাড়ে। অগ্রিম টাকা পাঠিয়ে বুকিং নিশ্চিত করুন, অফার আজই শেষ।"] - [bl, "eid offer! {item} ordhek dame, age taka pathan pore product, limited time"] + - [mx, "Mega sale! {item_bn} মাত্র half price এ, আজই full advance payment করে order confirm করুন।"] - id: phishing_kyc_link + typology: otp_pin_phishing labels: [phishing_malware, impersonation, account_takeover] templates: - [en, "Your {brand} account will be suspended. Update your KYC now: {bad}"] @@ -188,6 +231,8 @@ families: - [en, "Security alert: unusual login on your {brand} wallet. Verify your identity here {bad} within 24 hours."] - [bn, "{brand_bn} গ্রাহক, আপনার এনআইডি যাচাই বাকি আছে। লিংকে ক্লিক করে পিন দিয়ে লগইন করুন {bad}"] - [bl, "{brand} notice: apnar account limited kora hoyeche. unlock korte ekhane click korun {bad}"] + - [mx, "আপনার {brand} account এর KYC update করা হয়নি। এখনই এই link এ click করে PIN দিয়ে login করুন: {bad}"] + - [bl, "apnar {brand} acc 24 ghontar moddhe off hoye jabe, ekhane click kore info update koren {bad}"] - id: apk_install labels: [phishing_malware, impersonation] @@ -198,6 +243,7 @@ families: - [en, "Your wedding invitation card is attached, open the app file to view: {apk}"] - [bl, "play store e pawa jabe na, ei apk ta install koren {apk}, unknown source allow kore din"] - [bn, "বিদ্যুৎ বিলের নতুন অ্যাপ, এখান থেকে নামিয়ে ইনস্টল করুন {apk}, প্লে স্টোরে নেই।"] + - [mx, "{brand} এর new version এর app update করুন, এই link থেকে download করে install করুন {apk}"] - id: remote_access labels: [phishing_malware, social_engineering, account_takeover] @@ -208,6 +254,7 @@ families: - [en, "Download AnyDesk, open it and read me the 9 digit address so I can verify your {brand} app from our office."] - [bl, "screen share on koren whatsapp e, ami dekhe dekhe apnar {brand} thik kore dicchi"] - [bn, "হোয়াটসঅ্যাপে স্ক্রিন শেয়ার চালু করুন, তারপর আপনার {brand_bn} অ্যাপ খুলুন, আমি দেখে ঠিক করে দিচ্ছি।"] + - [mx, "আপনার app এর problem fix করতে AnyDesk install করুন, তারপর screen এ যে code আসবে আমাকে বলুন।"] - id: fake_qr labels: [phishing_malware, fake_payment] @@ -218,6 +265,7 @@ families: - [en, "To accept the refund, scan the attached QR and approve the request with your PIN."] - [bl, "ami payment request pathaisi, apni approve kore pin dile taka peye jaben"] - [bn, "আমি টাকা পাঠাচ্ছি, আপনার কাছে একটা রিকোয়েস্ট যাবে, পিন দিয়ে অ্যাপ্রুভ করলে টাকা পেয়ে যাবেন।"] + - [mx, "Money receive করতে এই QR code scan করে PIN দিন, সাথে সাথে টাকা চলে আসবে।"] - id: mule_recruit labels: [financial_network] @@ -228,6 +276,7 @@ families: - [bl, "amader company r payment receive korar jonno personal account lagbe, proti transaction e 5% commission, taka ele forward kore diben"] - [en, "We buy registered {brand} accounts and SIMs. Tk {amt} per account, payment on delivery. Inbox us."] - [bn, "আপনার এনআইডি দিয়ে খোলা {brand_bn} অ্যাকাউন্ট আমাদের ভাড়া দিন, প্রতি মাসে {amt_bn} টাকা পাবেন।"] + - [mx, "আপনার {brand} account rent দিন, monthly {amt_bn} টাকা income, শুধু SIM আর PIN দিলেই হবে।"] - id: betting_agent labels: [financial_network, scam_campaign] @@ -238,8 +287,10 @@ families: - [bl, "casino / cricket bet e taka lagaile double! deposit agent {num}, {brand} personal"] - [en, "Become a master agent for our gaming site. Receive player deposits in your wallet and earn commission daily."] - [bn, "ক্রিকেট বেটিংয়ে টাকা লাগান, দ্বিগুণ জিতুন। ডিপোজিট এজেন্ট নম্বর {num_bn}, শুধু সেন্ড মানি।"] + - [mx, "Online casino তে deposit করতে এই personal number এ cash in করুন {num_bn}, instant withdraw, 100% bonus।"] - id: lottery + typology: prize_lottery labels: [scam_campaign, social_engineering] templates: - [bl, "ovinondon! apni {big} taka lottery jitechen. taka pete registration fee {fee} taka ei number e pathan {num}"] @@ -248,8 +299,12 @@ families: - [bl, "apnar sim number ta {big} takar puroshkar peyeche, tax babod {fee} tk pathale taka apnar account e jabe"] - [bn, "আপনি একটি মোটরসাইকেল ও নগদ {big_bn} টাকা পুরস্কার জিতেছেন। রেজিস্ট্রেশন ফি {fee_bn} টাকা পাঠিয়ে পুরস্কার বুঝে নিন।"] - [en, "Lucky draw winner! You won a car and cash prize. Send the registration charge of Tk {fee} today or the prize goes to the next winner."] + - [mx, "Congratulations! আপনার number টি {brand} lucky draw তে {big_bn} টাকা জিতেছে। Claim করতে {fee_bn} টাকা processing fee পাঠান।"] + - [bl, "apnar mobile nambar ti {big} takar lottery te bijoyi hoyeche, puroshkar nite {fee} tk vat din {num}"] + - [en, "You are the winner of our mobile number raffle! Prize: Tk {big}. Send Tk {fee} delivery charge to {num} to receive it."] - id: investment_app + typology: job_investment labels: [scam_campaign] templates: - [bl, "amader app e {amt} taka invest korle protidin {pct}% profit, jokhon khushi withdraw. join korun"] @@ -258,8 +313,11 @@ families: - [bl, "forex / crypto trading e taka din, 30 din e double. amader team leader sob kore dibe, sudhu deposit koren"] - [en, "Join our VIP investment group. Deposit today, get double in 30 days. Withdraw anytime, no risk."] - [bn, "আমাদের ট্রেডিং অ্যাপে টাকা রাখলে ৩০ দিনে দ্বিগুণ। আজই ডিপোজিট করুন, টিম লিডার সব শিখিয়ে দেবে।"] + - [mx, "আমাদের trading app এ {amt_bn} টাকা invest করুন, daily {pct}% profit guaranteed, যেকোনো সময় withdraw।"] + - [bl, "crypto mining e taka dile 15 dine double, amader group e join koren, minimum deposit {amt} tk"] - id: task_job + typology: job_investment labels: [scam_campaign] templates: - [en, "Part-time job: like and subscribe YouTube videos and earn Tk {amt} daily. Message us on Telegram to start."] @@ -268,8 +326,12 @@ families: - [en, "You completed 3 tasks and earned Tk 150. To unlock the premium tasks, recharge Tk {fee} to our receptionist's number {num}."] - [bl, "telegram e join korun, hotel review dile daily {amt} tk income, age {fee} tk recharge kore account active korte hobe"] - [bn, "ডাটা এন্ট্রির চাকরি, কোনো অভিজ্ঞতা লাগবে না। রেজিস্ট্রেশন ফি {fee_bn} টাকা পাঠিয়ে আজই শুরু করুন।"] + - [mx, "Part time job: ঘরে বসে daily {amt_bn} টাকা income। শুরু করতে {fee_bn} টাকা registration fee লাগবে।"] + - [en, "Online typing job, earn Tk {amt} per day from your phone. Pay Tk {fee} for the training kit and start today."] + - [bl, "garments e chakri, beton {amt} taka, chakri confirm korte {fee} tk registration fee pathan {num} e"] - id: loan_fee + typology: job_investment labels: [scam_campaign] templates: - [bl, "5 minute e {big} taka loan, kono jamanot lagbe na. processing fee {fee} taka age pathan"] @@ -278,8 +340,10 @@ families: - [en, "Instant loan without documents. Send file opening charge Tk {fee} first."] - [bl, "apnar loan approve hoyeche, taka charte vat {fee} tk lagbe, ei number e pathan {num}"] - [bn, "সহজ কিস্তিতে ঋণ নিন। ফাইল খোলার চার্জ {fee_bn} টাকা এই নম্বরে আগে পাঠাতে হবে {num_bn}।"] + - [mx, "আপনার {big_bn} টাকার loan approve হয়েছে। disburse করতে {fee_bn} টাকা processing fee পাঠান {num_bn}।"] - id: cashback_bait + typology: prize_lottery labels: [scam_campaign, impersonation] templates: - [bl, "{brand} er pokkho theke apni {amt} taka cashback peyechen, pete hole age {fee} taka send money korun"] @@ -288,8 +352,10 @@ families: - [bl, "offer! {fee} taka pathale {amt} taka ferot, {brand} eid bonus, sudhu ajker jonno"] - [en, "You have been selected for a Tk {amt} bonus. Activate it by sending Tk {fee} to our offer number."] - [bn, "আপনি {amt_bn} টাকা বোনাসের জন্য নির্বাচিত হয়েছেন। বোনাস চালু করতে {fee_bn} টাকা অফার নম্বরে পাঠান।"] + - [mx, "{brand} Eid offer: মাত্র {fee_bn} টাকা send করলে instant {amt_bn} টাকা cashback, শুধু আজকের জন্য!"] - id: remittance_bonus + typology: prize_lottery labels: [scam_campaign, impersonation] templates: - [bn, "বিদেশ থেকে আপনার নামে {big_bn} টাকা রেমিট্যান্স এসেছে। টাকা ছাড়াতে {fee_bn} টাকা ট্যাক্স পাঠাতে হবে।"] @@ -298,6 +364,7 @@ families: - [en, "Bangladesh Bank notice: your foreign remittance is on hold. Send the service charge to release the funds."] - [bl, "apnar probashi vai taka pathaiche, kintu limit er jonno atke ase, {fee} tk diye limit barate hobe"] - [bn, "বাংলাদেশ ব্যাংক থেকে বলছি, আপনার প্রবাসী আত্মীয়ের পাঠানো টাকা আটকে আছে। ছাড়াতে সার্ভিস চার্জ পাঠান।"] + - [mx, "আপনার নামে Dubai থেকে {big_bn} টাকা remittance এসেছে, release করতে {fee_bn} টাকা tax send করুন।"] # ------------------------------------------ scams the model never trains on - id: customs_parcel @@ -311,6 +378,7 @@ families: - id: bank_card held_out: true + typology: otp_pin_phishing labels: [impersonation, social_engineering, account_takeover] templates: - [bl, "ami bank theke bolchi, apnar ATM card block hoye gese, card er 16 digit number ar pichoner 3 digit bolun"] @@ -320,6 +388,7 @@ families: - id: overpayment held_out: true + typology: wrong_number labels: [fake_payment] templates: - [bl, "vai ami 500 er jaygay {amt} pathiye disi vul e, baki taka ta back kore den"] @@ -329,6 +398,7 @@ families: - id: esim_porting held_out: true + typology: otp_pin_phishing labels: [account_takeover] templates: - [en, "eSIM conversion request received for your number. If this was you, share the activation code with our agent."] @@ -373,6 +443,72 @@ families: - [en, "Scholarship approved for your child. Deposit the admission charge today to this number to keep the seat."] - [bl, "bidesh e chakri, visa ready, sudhu {fee} tk medical fee pathan aaj-i"] + - id: helpline_search + held_out: true + typology: fake_agent_helpline + labels: [impersonation, social_engineering] + templates: + - [en, "Hi, I am replying from the {brand} support page about your post. Your money can be recovered. Message our agent on WhatsApp at {num} with your wallet number."] + - [bn, "আপনার পোস্টটি দেখলাম। আমি {brand_bn} রিকভারি টিমের সদস্য, ভুল নম্বরে যাওয়া টাকা ফেরত আনা যাবে। ইমোতে নক দিন {num_bn}।"] + - [bl, "google e je {brand} hotline number paisen oita amader, apnar taka uddhar korte {num} e imo te knock din"] + - [mx, "আপনার comment টা দেখেছি, {brand} support team থেকে বলছি। Wrong transaction recover করতে inbox এ wallet number দিন।"] + - [en, "Official {brand} agent desk: we fix stuck cash in, failed transfers and locked wallets. Inbox your number and the last balance."] + + - id: prize_quiz + held_out: true + typology: prize_lottery + labels: [scam_campaign, social_engineering] + templates: + - [en, "You answered our TV quiz correctly and won a motorcycle. Pay the registration and transport cost of Tk {fee} to {num} to receive it."] + - [bn, "টেলিভিশন কুইজে সঠিক উত্তর দিয়ে আপনি একটি ফ্রিজ জিতেছেন। পুরস্কার পাঠাতে পরিবহন খরচ {fee_bn} টাকা পাঠান।"] + - [bl, "apni amader facebook page er quiz e first hoyechen, gift paben ekta smart TV, delivery charge {fee} tk pathan"] + - [mx, "আপনার SIM number টি mobile operator এর anniversary raffle এ selected হয়েছে, gift পেতে {fee_bn} টাকা VAT দিন।"] + - [en, "Scratch card lucky winner! Your code matched our grand prize of Tk {big}. Customs and VAT of Tk {fee} must be paid first."] + + - id: wrong_recharge + held_out: true + typology: wrong_number + labels: [fake_payment, social_engineering] + templates: + - [bl, "vai vul kore apnar number e {amt} taka flexiload chole gese, oita ektu send money kore den {num} e"] + - [bn, "ভাই, ভুল করে আপনার নম্বরে {amt_bn} টাকা রিচার্জ চলে গেছে। টাকাটা এই নম্বরে বিকাশ করে দিন, খুব উপকার হবে।"] + - [en, "I recharged Tk {amt} to your number by mistake. Please send it back by mobile banking to {num}, my mother needs it."] + - [mx, "Sorry ভাই, ভুলে আপনার SIM এ {amt_bn} টাকা recharge হয়ে গেছে, please টাকাটা এই number এ send money করে দেন।"] + - [bl, "apu amar recharge ta vul e apnar sim e gese, balance check koren, {amt} tk amar {brand} e pathiye den pls"] + + - id: job_deposit + held_out: true + typology: job_investment + labels: [scam_campaign] + templates: + - [en, "Security guard jobs available at the airport, salary Tk {amt}. Send the uniform and ID card fee of Tk {fee} to {num} to secure your post."] + - [bn, "সরকারি হাসপাতালে আউটসোর্সিং পদে নিয়োগ নিশ্চিত। নিয়োগপত্র পেতে {fee_bn} টাকা জামানত এই নম্বরে পাঠান।"] + - [bl, "amader company te home delivery rider lagbe, joining er age {fee} tk security deposit pathate hobe {num} e"] + - [mx, "Daily {amt_bn} টাকা income এর জন্য আমাদের mining app এ deposit করুন, 7 দিনে double profit, refer করলে extra bonus।"] + - [en, "Gold investment club: put in Tk {amt} today and receive Tk {amt} profit every week. Only 20 seats left."] + + - id: govt_aid_card + held_out: true + typology: govt_aid + labels: [impersonation, scam_campaign, social_engineering] + templates: + - [bn, "টিসিবির ফ্যামিলি কার্ডের তালিকায় আপনার নাম এসেছে। কার্ড সচল করতে {fee_bn} টাকা ফি পাঠান {num_bn} নম্বরে।"] + - [bl, "apnar nam family card er list e ase, card active korte {fee} taka ei number e pathan {num}, ajker moddhe"] + - [en, "Government flood relief: Tk {amt} has been allocated to your family. Share the code sent to your phone with the union officer to receive it."] + - [mx, "মাতৃত্বকালীন ভাতার {amt_bn} টাকা approve হয়েছে, পেতে আপনার {brand} PIN টা officer কে বলুন।"] + - [bn, "জেলা প্রশাসকের কার্যালয় থেকে বলছি, ঘূর্ণিঝড়ে ক্ষতিগ্রস্তদের তালিকায় আপনার নাম আছে। সহায়তার টাকা পেতে মোবাইলে আসা কোডটি জানান।"] + + - id: otp_social + held_out: true + typology: otp_pin_phishing + labels: [account_takeover, social_engineering] + templates: + - [en, "Hi, it's me from the family WhatsApp group. I sent you a 6-digit code by mistake, can you send it back to me quickly?"] + - [bl, "dost amar imo te login korte parchi na, tor number e ekta code gese, oita amake whatsapp e de ekhoni"] + - [bn, "আমি ফেসবুক সাপোর্ট থেকে বলছি, আপনার পেজ বন্ধ হয়ে যাবে। যাচাই করতে আপনার ফোনে আসা কোডটি এখানে লিখুন।"] + - [mx, "আপনার WhatsApp account verify করতে হবে, SMS এ যে 6 digit code গেছে সেটা reply করুন, না হলে account ban হবে।"] + - [en, "Your Google account recovery was requested. If you want to cancel it, reply with the verification code you just received."] + # --------------------------------------------------------------- harmless - id: real_otp hard: true @@ -384,6 +520,7 @@ families: - [bl, "apnar {brand} OTP {otp}. ei code karo sathe share korben na."] - [en, "Use {otp} to log in to your account. If you did not request this, ignore this message."] - [bn, "{otp} আপনার লগইন কোড। আপনি অনুরোধ না করে থাকলে এই বার্তা উপেক্ষা করুন।"] + - [mx, "আপনার {brand} OTP {otp}। এই code কারো সাথে share করবেন না।"] - id: real_txn_sms labels: [] @@ -395,6 +532,7 @@ families: - [en, "Payment Tk {amt}.00 to merchant successful. Balance Tk {fee}.20. TrxID {trx}"] - [bn, "ক্যাশ আউট {amt_bn} টাকা সফল হয়েছে। চার্জ {fee_bn} টাকা। TrxID {trx}"] - [en, "Cash Out Tk {amt}.00 to agent {num} successful. Fee Tk {fee}.00. TrxID {trx}"] + - [mx, "Cash In {amt_bn} টাকা successful। Balance {amt_bn} টাকা। TrxID {trx}"] - id: awareness hard: true @@ -408,6 +546,7 @@ families: - [bl, "keu vul kore taka pathaise bolle age nijer balance check korun, tarpor siddhanto nin"] - [en, "Reminder: check your own balance before returning any money sent by mistake. A screenshot is not proof."] - [bn, "অপরিচিত লিংকে ক্লিক করবেন না এবং অচেনা কাউকে কোড বলবেন না। নিরাপদ থাকুন।"] + - [mx, "সতর্ক থাকুন: {brand} কখনো আপনার PIN বা OTP জানতে চায় না, কাউকে share করবেন না।"] - id: family_money hard: true @@ -420,6 +559,12 @@ families: - [bn, "আপা, আম্মুর ওষুধ কিনতে হবে, সন্ধ্যায় {amt_bn} টাকা পাঠাতে পারবে?"] - [en, "Hey, I sent you the rent share, {amt} taka. Check when you're free."] - [bl, "dost kal je {amt} taka dhar nisilam ajke pathiye dilam, dekhis"] + - [mx, "Ammu এই month এর mess এর টাকা {amt_bn} পাঠিও, no hurry, কাল হলেও চলবে।"] + - [bl, "ammu ei mash er hostel er bhara {amt} taka bkash kore dio, kal dileo hobe"] + - [bl, "apu {amt} tk pathaba? boi kinte hobe, tomar subidha moto dio"] + - [bl, "bhaiya amar number e {amt} taka bkash koro, bari ashle ferot dibo"] + - [mx, "আব্বু hostel এর bill {amt_bn} টাকা bkash করে দিও, সময় মতো দিলেই হবে।"] + - [bl, "mama, ammu bollo {amt} taka lagbe, somoy pele amar number e pathiye diben"] - id: chat labels: [] @@ -432,6 +577,8 @@ families: - [en, "Happy birthday! Hope you have a great day with the family."] - [bl, "khela dekhsos? bangladesh jitse aaj"] - [en, "The meeting is moved to Sunday morning, please bring the printed report."] + - [mx, "আজকে class হবে না, sir notice দিয়েছেন। কাল lab report submit করতে হবে।"] + - [mx, "Birthday party তে আসছিস তো? Saturday সন্ধ্যা ৭টায়।"] - id: promo hard: true @@ -443,6 +590,7 @@ families: - [en, "Pay at partner supermarkets with {brand} and enjoy 10% instant discount this weekend."] - [bn, "এই ঈদে {brand_bn} পেমেন্টে কেনাকাটায় ১০% ছাড়। নির্দিষ্ট আউটলেটে প্রযোজ্য।"] - [en, "New in the {brand} app: pay your tuition fees with no charge. Update from the app store."] + - [mx, "{brand} app দিয়ে bill pay করলে 10% cashback, offer চলবে এই month পর্যন্ত।"] - id: shop_order hard: true @@ -455,6 +603,10 @@ families: - [en, "Your order has been shipped. Track it here: {web}. Pay the rider when you receive it."] - [bn, "{item_bn} স্টকে আছে, দাম {amt_bn} টাকা। দোকানে এসে দেখে নিতে পারেন, ঠিকানা ইনবক্সে দিচ্ছি।"] - [en, "You can pay through our merchant account at checkout or choose cash on delivery, whichever you prefer."] + - [mx, "আপনার order confirm হয়েছে, cash on delivery। Rider delivery এর আগে call দিবে।"] + - [en, "Your parcel with {item} is out for delivery today. Order details: {web}. Pay cash to the rider."] + - [bl, "apnar parcel rowna hoyeche, details {web} e dekhen, taka rider ke hate diben"] + - [en, "Order shipped! See your order here: {web}. Nothing to pay now, cash on delivery."] - id: job_post hard: true @@ -466,6 +618,7 @@ families: - [en, "Your interview is scheduled for Tuesday. Please confirm attendance by replying to this message."] - [bn, "আপনার আবেদন গ্রহণ করা হয়েছে। লিখিত পরীক্ষার তারিখ পরে জানানো হবে।"] - [en, "Internship opening for students, stipend Tk {amt} per month. Apply through our website {web}."] + - [mx, "আমাদের office এ একজন accountant লাগবে, interview Sunday, কোনো fee লাগবে না।"] - id: salary_bill labels: [] @@ -477,6 +630,7 @@ families: - [bn, "আপনার মোবাইল রিচার্জ {fee_bn} টাকা সফল হয়েছে।"] - [bl, "gas bill {amt} taka porishodh hoyeche, dhonnobad"] - [en, "Tuition fee received: Tk {amt}. Receipt number {trx}."] + - [mx, "আপনার এই মাসের salary {amt_bn} টাকা wallet এ পাঠানো হয়েছে।"] - id: honest_buyer hard: true @@ -488,6 +642,7 @@ families: - [bl, "apu taka dhukse naki dekhen to, na dhukle bolben ami abar check korbo"] - [en, "Sent the money just now. Let me know once you see it in your balance."] - [bn, "পেমেন্ট করেছি, আপনার ব্যালেন্সে এসেছে কিনা দেখে জানাবেন।"] + - [mx, "Payment করেছি {amt_bn} টাকা, আপনার app এ check করে confirm করেন, তারপর পাঠাবেন।"] - id: support_question hard: true @@ -500,6 +655,7 @@ families: - [bl, "amar phone hariye gese, wallet ta kivabe bondho korbo?"] - [bn, "একজন ফোন করে ওটিপি চাইছিল, আমি দিইনি। এটা কোথায় জানাব?"] - [en, "Someone called asking for my OTP and I refused. Where can I report this number?"] + - [mx, "আমি ভুল করে অন্য number এ টাকা send করে ফেলেছি, কীভাবে complaint করব?"] - id: everyday_notices labels: [] @@ -512,6 +668,35 @@ families: - [bn, "আপনার ডাক্তারের অ্যাপয়েন্টমেন্ট শনিবার সন্ধ্যা ৬টায়।"] - [en, "Watch the highlights here {web}"] - [bl, "ei video ta dekh {web} onek mojar"] + - [mx, "আপনার parcel আজ delivery হবে, rider call দিবে।"] + + - id: real_govt_payment + hard: true + labels: [] + templates: + - [en, "You have received Tk {amt} allowance from the Department of Social Services. Fee Tk 0. Balance Tk {amt}.00. TrxID {trx}"] + - [bn, "উপবৃত্তির {amt_bn} টাকা আপনার {brand_bn} অ্যাকাউন্টে জমা হয়েছে। TrxID {trx}। কেউ পিন বা কোড চাইলে দেবেন না।"] + - [bl, "apnar boyosko vatar {amt} taka account e joma hoyeche, nikotsto agent theke cash out korte parben"] + - [mx, "আপনার stipend এর {amt_bn} টাকা account এ জমা হয়েছে। কোনো fee লাগবে না, কেউ PIN চাইলে দেবেন না।"] + - [en, "Your TCB family card is ready for collection at the union parishad office. Bring your NID. No fee is charged."] + + - id: real_helpline + hard: true + labels: [] + templates: + - [en, "For any help with your {brand} account call the official helpline 16247 from your registered number. We never ask for your PIN."] + - [bn, "যেকোনো প্রয়োজনে আমাদের অফিসিয়াল হেল্পলাইন ১৬১৬৭ নম্বরে কল করুন। কেউ পিন চাইলে দেবেন না।"] + - [bl, "apnar complaint ti grohon kora hoyeche, ticket no {trx}. amader agent official 16216 number theke call korbe"] + - [mx, "আপনার complaint টি নেওয়া হয়েছে, ticket {trx}। App এর Help section থেকে status দেখতে পারবেন।"] + + - id: real_prize + hard: true + labels: [] + templates: + - [en, "Congratulations on winning the school debate competition! Collect your certificate from the office on Sunday."] + - [bn, "অভিনন্দন! আপনি আমাদের রিচার্জ ক্যাম্পেইনে ৫০ টাকা ক্যাশব্যাক পেয়েছেন, যা ইতিমধ্যে আপনার অ্যাকাউন্টে যোগ হয়েছে।"] + - [bl, "ovinondon! apni cashback offer e 20 taka peyechen, eti apnar balance e add kora hoyeche. kono kichu pathate hobe na"] + - [mx, "Congratulations! আপনার {brand} cashback {fee_bn} টাকা already balance এ add হয়ে গেছে।"] # --------------------------------- harmless text the model never trains on - id: school_notice @@ -522,6 +707,7 @@ families: - [en, "Parents meeting on Thursday at 11am. Monthly fee can be paid at the school office or through the bank."] - [bl, "kal school bondho, porikkhar routine pore janano hobe"] - [en, "Exam results are published on the notice board."] + - [mx, "কাল school বন্ধ থাকবে, exam এর new routine পরে জানানো হবে।"] - id: split_bill held_out: true @@ -532,6 +718,7 @@ families: - [en, "Dinner was {amt} total, your share is {fee}. Send it whenever, my number is the same as always."] - [bn, "গতকালের পিকনিকের চাঁদা {fee_bn} টাকা, সুবিধামতো আমাকে পাঠিয়ে দিও।"] - [en, "I'll pay you back the {amt} taka on Friday when I get my salary."] + - [mx, "কালকের dinner এর bill এ তোর share {fee_bn} টাকা, সময় করে পাঠাস।"] - id: official_notice held_out: true @@ -542,3 +729,4 @@ families: - [bn, "প্রিয় গ্রাহক, আপনার পিন পরিবর্তন সফল হয়েছে। আপনি এটি না করে থাকলে অ্যাপের হেল্পলাইনে যোগাযোগ করুন।"] - [bl, "apnar {brand} app e notun device theke login hoyeche. apni na hole app theke pin change korun"] - [en, "Your KYC information was updated successfully at our customer care centre. No further action is needed."] + - [mx, "আপনার {brand} app এ new device থেকে login হয়েছে। আপনি না হলে app থেকে PIN change করুন।"] diff --git a/backend/src/fraudlens/intel/cues.yaml b/backend/src/fraudlens/intel/cues.yaml index faf0b54..e43228e 100644 --- a/backend/src/fraudlens/intel/cues.yaml +++ b/backend/src/fraudlens/intel/cues.yaml @@ -4,7 +4,8 @@ # # Patterns are regular expressions matched on normalised text: lower case, every # digit written as 0, every link written as `urltoken`. They cover Bangla script, -# Bangla in Latin letters (spelling varies, so the patterns are loose) and English. +# Bangla in Latin letters (spelling varies, so the patterns are loose), English, +# and the two scripts mixed in one message ("OTP টা বলুন"). # # `benign: true` marks a cue that points away from a scam (a warning not to share # a code, a transaction receipt). Those are features too, but never shown as a reason. @@ -20,6 +21,8 @@ cues: - '(?:otp|\bpin\b|passw|passcode|\bc[o0]de?\b|\bkod\b|confirmation number|0 digit|\bid shown)[^.!?।]{0,60}(?:bol[ue]n|bolben|\bbolo\b|\bd[ie]n\b|diben|\bdao\b|janan|forward|share|\btell\b|\bgive\b|reply|read me|enter|provide|approve)' - '(?:\btell\b|\bgive\b|share|forward|reply with|read me|confirm your|enter your|provide)[^.!?।]{0,40}(?:otp|\bpin\b|passw|passcode|\bcode\b|digit|\bid\b)' - '(?:ওটিপি|পিন|কোড|পাসওয়ার্ড|নম্বর আসবে)[^.!?।]{0,50}(?:বলুন|বলবেন|দিন|জানান|পাঠান|বলো|দাও)' + # Both scripts at once: an English word for the secret, a Bangla verb for giving it. + - '(?:otp|\bpin\b|passw|\bc[o0]dee?\b)[^.!?।]{0,50}(?:বলুন|বলবেন|দিন|দেন|জানান|পাঠান|বলো|দাও|শেয়ার)' - id: warn_secret benign: true @@ -78,6 +81,7 @@ cues: - 'agrim|jamanot ?(?:hisebe|den|din)|full payment' - '(?:ফি|চার্জ|ট্যাক্স|ভ্যাট|অগ্রিম|জমা|ডিউটি)[^.!?।]{0,50}(?:পাঠান|পাঠাতে|পাঠালে|পাঠিয়ে|দিতে হবে|দিন|দিয়ে|লাগবে)' - 'আগে[^.!?।]{0,40}(?:টাকা|চার্জ|ফি)[^.!?।]{0,30}(?:পাঠ|দিতে|জমা)' + - '(?:\bfee\b|charge|\btax\b|\bvat\b|deposit|advance|clearance)[^.!?।]{0,50}(?:পাঠান|পাঠাতে|পাঠালে|দিতে হবে|দিন|দেন|লাগবে|করুন)' - id: return_money label: diff --git a/backend/src/fraudlens/intel/explain.py b/backend/src/fraudlens/intel/explain.py new file mode 100644 index 0000000..d790405 --- /dev/null +++ b/backend/src/fraudlens/intel/explain.py @@ -0,0 +1,155 @@ +"""Which words in the customer's own message made the classifier say so. + +Two kinds of evidence, both pointing at spans of the text the customer pasted, in +whatever script it was written: + +- **phrases**: for each cue that matched (`cues.yaml`), the exact stretch of text + the pattern matched ("OTP টা বলুন", "pin ta bolen", "registration fee"). +- **highlights**: the words that pushed the scam score up most. The character + n-grams the model reads never cross a word boundary (`char_wb`), so each n-gram's + weight belongs to exactly one word; a word's share is the sum of its n-grams'. + +Nothing is generated: both are read off the patterns and the model's weights. +""" + +from __future__ import annotations + +import re +import unicodedata +from collections import Counter + +from sklearn.feature_extraction.text import TfidfVectorizer + +from .cues import Cue, matched +from .links import URL + +MAX_CHARS = 2000 +MAX_PHRASES = 3 # per cue +MAX_HIGHLIGHTS = 5 + +_BN_DIGITS = str.maketrans("০১২৩৪৫৬৭৮৯", "0123456789") +_DIGIT = re.compile(r"\d") + + +def mapped(text: str) -> tuple[str, list[int], list[int]]: + """`text.normalise`, one character at a time, keeping where each output + character came from: (normalised, start, end), end exclusive, into `text`.""" + text = text[:MAX_CHARS] + chars: list[str] = [] + starts: list[int] = [] + ends: list[int] = [] + for i, ch in enumerate(text): + for c in unicodedata.normalize("NFKC", ch).translate(_BN_DIGITS).lower(): + chars.append(c) + starts.append(i) + ends.append(i + 1) + joined = "".join(chars) + + # Links become one token, as the model sees them; the token points at the whole link. + out_c: list[str] = [] + out_s: list[int] = [] + out_e: list[int] = [] + last = 0 + for m in URL.finditer(joined): + a, b = m.span() + out_c.extend(joined[last:a]) + out_s.extend(starts[last:a]) + out_e.extend(ends[last:a]) + token = " urltoken " + out_c.extend(token) + out_s.extend([starts[a]] * len(token)) + out_e.extend([ends[b - 1]] * len(token)) + last = b + out_c.extend(joined[last:]) + out_s.extend(starts[last:]) + out_e.extend(ends[last:]) + + # Digits to 0, runs of space to one, no space at either end. + norm_c: list[str] = [] + norm_s: list[int] = [] + norm_e: list[int] = [] + for c, s, e in zip(out_c, out_s, out_e, strict=True): + if c.isspace(): + if not norm_c or norm_c[-1] == " ": + continue + c = " " + norm_c.append(_DIGIT.sub("0", c)) + norm_s.append(s) + norm_e.append(e) + while norm_c and norm_c[-1] == " ": + norm_c.pop() + norm_s.pop() + norm_e.pop() + return "".join(norm_c), norm_s, norm_e + + +def _span(text: str, starts: list[int], ends: list[int], a: int, b: int) -> dict: + lo, hi = starts[a], ends[b - 1] + return {"text": text[lo:hi].strip(), "start": lo, "end": hi} + + +def cue_phrases(text: str) -> list[tuple[Cue, list[dict]]]: + """Each scam cue the message shows, with the stretches of text that showed it.""" + norm, starts, ends = mapped(text) + out = [] + for cue in matched(norm): + if cue.benign: + continue + found: list[dict] = [] + for pattern in cue.patterns: + for m in pattern.finditer(norm): + if m.end() > m.start(): + found.append(_span(text, starts, ends, m.start(), m.end())) + found.sort(key=lambda p: p["start"]) + unique: list[dict] = [] + for phrase in found: + if phrase["text"] and phrase not in unique: + unique.append(phrase) + out.append((cue, unique[:MAX_PHRASES])) + return out + + +def _chars_and_weights(pipeline) -> tuple[TfidfVectorizer, float, object] | None: + """The character n-gram vectorizer, its weight in the feature union, and the + scam output's coefficients for those columns; None for any other kind of model.""" + try: + union = pipeline.named_steps["features"] + name, vectorizer = union.transformer_list[0] + head = pipeline.named_steps["model"].estimators_[0] + except (AttributeError, KeyError, IndexError): + return None + if not isinstance(vectorizer, TfidfVectorizer) or vectorizer.analyzer != "char_wb": + return None + weight = (union.transformer_weights or {}).get(name, 1.0) + coef = head.coef_[0][: len(vectorizer.vocabulary_)] + return vectorizer, weight, coef + + +def highlights(text: str, pipeline) -> list[dict]: + """The words that raised the scam score most, strongest first.""" + parts = _chars_and_weights(pipeline) + if parts is None: + return [] + vectorizer, weight, coef = parts + norm, starts, ends = mapped(text) + row = vectorizer.transform([norm]) + value = dict(zip(row.indices.tolist(), row.data.tolist(), strict=True)) + analyse = vectorizer.build_analyzer() + vocab = vectorizer.vocabulary_ + + words = [(m.start(), m.end()) for m in re.finditer(r"\S+", norm)] + grams = [Counter(g for g in analyse(norm[a:b]) if g in vocab) for a, b in words] + total: Counter = Counter() + for counts in grams: + total.update(counts) + + scored = [] + for (a, b), counts in zip(words, grams, strict=True): + score = sum( + weight * value.get(vocab[g], 0.0) * coef[vocab[g]] * n / total[g] + for g, n in counts.items() + ) + if score > 0.01: + scored.append(_span(text, starts, ends, a, b) | {"weight": round(float(score), 3)}) + scored.sort(key=lambda w: -w["weight"]) + return scored[:MAX_HIGHLIGHTS] diff --git a/backend/src/fraudlens/intel/external.py b/backend/src/fraudlens/intel/external.py new file mode 100644 index 0000000..c483dd4 --- /dev/null +++ b/backend/src/fraudlens/intel/external.py @@ -0,0 +1,124 @@ +"""A public, human-labelled smishing corpus, used to train and to test on text we did not write. + +Source: "Bengali SMS Smishing Dataset" (Hugging Face +`shariul-islam/bengali-sms-smishing-dataset`, MIT licence), 7,005 SMS in Bangla +script, Banglish, English and code-mixed Bangla-English, each labelled `smish`, +`promo` or `normal` by its authors. It is fetched at a pinned revision and checked +against fixed SHA-256 sums, so the same bytes are used every time; nothing from it +is committed to this repository. + +How it is used: `smish` is a scam, `promo` and `normal` are harmless. It has no +category labels, so it informs only the scam-or-not output, never the eight +categories. Its own train/validation/test split is kept: its test split is never +trained on and is the one external measurement in the report. + +The files are third-party data. They are only ever parsed as Parquet tables of +strings; nothing in them is executed. +""" + +from __future__ import annotations + +import hashlib +import logging +from dataclasses import dataclass +from pathlib import Path + +from ..config import Settings + +log = logging.getLogger(__name__) + +REPO = "shariul-islam/bengali-sms-smishing-dataset" +REVISION = "9d7c131cc63187c54dfd069d8070776c82f0be45" +LICENCE = "MIT" +URL = f"https://huggingface.co/datasets/{REPO}" +FILES = { + "train": ("data/train-00000-of-00001.parquet", + "9fb92ea7e185297f4dd66442a7ed5f11ffc82d501c1ca2ea38385f430b9fe18e"), + "val": ("data/validation-00000-of-00001.parquet", + "a6a652087bbc328618cc39720a3ec66be53e106d9184dcf83791bc4da705a1fc"), + "test": ("data/test-00000-of-00001.parquet", + "2719d61aaf82b4b38e256e7e0d5ecee4a19588a7f522b863c713a435d0dffa31"), +} # fmt: skip +# The dataset's names for how a message is written, mapped onto ours. +VARIETIES = {"Bengali": "bn", "Banglish": "bl", "English": "en", "CodeMix": "mx"} + + +@dataclass(frozen=True) +class External: + text: str + scam: bool + lang: str # bn | bl | en | mx + label: str # the dataset's own label: smish | promo | normal + + +def external_dir(settings: Settings) -> Path: + return settings.data_dir / "external" / "bengali-sms-smishing" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def fetch(directory: Path, timeout: float = 60.0) -> bool: + """Download the pinned files if they are not here yet. False when it cannot.""" + import httpx + + directory.mkdir(parents=True, exist_ok=True) + for split, (name, digest) in FILES.items(): + path = directory / f"{split}.parquet" + if path.exists() and _sha256(path) == digest: + continue + url = f"{URL}/resolve/{REVISION}/{name}" + try: + response = httpx.get(url, timeout=timeout, follow_redirects=True) + response.raise_for_status() + except httpx.HTTPError as exc: + log.warning("could not fetch %s: %s", url, exc) + return False + tmp = path.with_suffix(".part") + tmp.write_bytes(response.content) + if _sha256(tmp) != digest: + tmp.unlink() + log.warning("%s does not match its pinned checksum; not used", url) + return False + tmp.replace(path) + return True + + +def load(directory: Path) -> dict[str, list[External]] | None: + """The three splits, or None when the files are missing or not the pinned ones.""" + import pandas as pd + + out: dict[str, list[External]] = {} + for split, (_, digest) in FILES.items(): + path = directory / f"{split}.parquet" + if not path.exists() or _sha256(path) != digest: + return None + frame = pd.read_parquet(path, columns=["label", "text", "source"]) + out[split] = [ + External(str(text), label == "smish", VARIETIES[source], str(label)) + for label, text, source in frame.itertuples(index=False) + if source in VARIETIES and isinstance(text, str) and text.strip() + ] + return out + + +def ensure(settings: Settings) -> dict[str, list[External]] | None: + """The public corpus, fetched first if it is not here. None when offline: the + model then trains on the written corpus alone and the report says so.""" + directory = external_dir(settings) + found = load(directory) + if found is None and fetch(directory): + found = load(directory) + return found + + +def provenance() -> dict: + return { + "name": "Bengali SMS Smishing Dataset", + "url": URL, + "revision": REVISION, + "licence": LICENCE, + "files": {split: {"path": n, "sha256": d} for split, (n, d) in FILES.items()}, + "labels": "smish = scam; promo and normal = harmless", + } diff --git a/backend/src/fraudlens/intel/memory.py b/backend/src/fraudlens/intel/memory.py new file mode 100644 index 0000000..f60ba73 --- /dev/null +++ b/backend/src/fraudlens/intel/memory.py @@ -0,0 +1,159 @@ +"""A flagged message, then a payment: the two read together. + +A customer who asks "is this a scam?" and is told yes or maybe, and then pays the +wallet that message named, or the amount it asked for, within half an hour, is +doing what the message told them to. Each signal is weak alone; together they are +the scam. `MessageMemory` keeps just enough of each flagged message to see it: + +- the wallet IDs, phone numbers and amounts the message named; +- the level and categories it was given; when it was checked. + +Never the text. Entries live in this process only, for `WINDOW` minutes, a few per +wallet, and are gone on restart: this is a short reminder, not a record. + +`link` is what the scorer asks for each payment; `explain` is the reason shown. +""" + +from __future__ import annotations + +import re +import threading +import unicodedata +from collections import defaultdict, deque +from dataclasses import dataclass, replace +from datetime import datetime, timedelta + +WINDOW = timedelta(minutes=30) +PER_WALLET = 5 +AMOUNT_TOLERANCE = 1.0 # taka + +_BN_DIGITS = str.maketrans("০১২৩৪৫৬৭৮৯", "0123456789") +_WALLET = re.compile(r"\b([A-Za-z]\d{4,})\b") +_PHONE = re.compile(r"(? tuple[frozenset[str], frozenset[str], tuple[float, ...]]: + """The wallet IDs, phone numbers and amounts a message names.""" + text = unicodedata.normalize("NFKC", text).translate(_BN_DIGITS) + wallets = frozenset(m.upper() for m in _WALLET.findall(text)) + phones = frozenset(_PHONE.findall(text)) + amounts = [] + for m in _AMOUNT.finditer(text): + try: + value = float((m.group(1) or m.group(2)).replace(",", "")) + except ValueError: + continue + if value > 0: + amounts.append(value) + return wallets, phones, tuple(dict.fromkeys(amounts)) + + +class MessageMemory: + def __init__(self) -> None: + self._lock = threading.Lock() + self._by_wallet: dict[str, deque[Flagged]] = defaultdict(lambda: deque(maxlen=PER_WALLET)) + + def remember(self, wallet_id: str, text: str, found: dict, now: datetime) -> bool: + """Keep what a checked message named, if it was flagged and named anything.""" + if found.get("level", "none") == "none": + return False + wallets, phones, amounts = targets(text) + if not (wallets or phones or amounts): + return False + entry = Flagged( + now, found["level"], tuple(c["id"] for c in found.get("categories", ())), + wallets, phones, amounts, + ) # fmt: skip + with self._lock: + self._by_wallet[wallet_id].append(entry) + return True + + def link( + self, sender_id: str, receiver_id: str, amount: float | None, now: datetime + ) -> dict | None: + """The most recent flagged message this payment follows, and how it matches.""" + with self._lock: + entries = list(self._by_wallet.get(sender_id, ())) + for entry in reversed(entries): + age = now - entry.at + if age < timedelta(0) or age > WINDOW: + continue + matched = [] + if receiver_id.upper() in entry.wallets: + matched.append("receiver") + if amount is not None and any( + abs(amount - a) <= AMOUNT_TOLERANCE for a in entry.amounts + ): + matched.append("amount") + if matched: + return { + "matched": matched, + "minutes_ago": int(age.total_seconds() // 60), + "level": entry.level, + "categories": list(entry.categories), + } + return None + + def forget(self, wallet_id: str) -> None: + with self._lock: + self._by_wallet.pop(wallet_id, None) + + +def escalate(decision, link: dict, policy): + """The decision with the message link added: an allowed payment becomes at least + a warning (the customer still decides), an alert keeps its tier and gains the reason.""" + reason = explain(link) + evidence = dict(decision.evidence) | {"message_link": link} + changes: dict = { + "reasons": (reason, *decision.reasons), + "evidence": evidence, + } + if decision.tier == "allow": + changes |= { + "tier": "warn", + "action": policy.tiers["warn"].action, + "scenario": "scam", + "customer_message": policy.messages["scam"]["warn"].model_dump(), + "decided_by": "message_link", + } + return replace(decision, **changes) + + +def explain(link: dict) -> dict: + """The reason shown with the payment, in the shape the decision engine uses.""" + what_en = " and ".join( + {"receiver": "the wallet", "amount": "the amount"}[m] for m in link["matched"] + ) + what_bn = " ও ".join({"receiver": "ওয়ালেট", "amount": "টাকার পরিমাণ"}[m] for m in link["matched"]) + minutes = link["minutes_ago"] + return { + "code": "follows_flagged_message", + "source": "message", + "direction": "raises", + "title_en": "Follows a message flagged as a likely scam", + "title_bn": "সম্ভাব্য প্রতারণার বার্তার পরেই এই পেমেন্ট", + "detail_en": ( + f"{minutes} minutes ago you checked a message that looked like a scam; " + f"this payment goes to {what_en} it named" + ), + "detail_bn": ( + f"{minutes} মিনিট আগে যাচাই করা একটি বার্তা প্রতারণার মতো ছিল; " + f"এই পেমেন্ট সেই বার্তায় লেখা {what_bn} অনুযায়ী যাচ্ছে" + ), + } diff --git a/backend/src/fraudlens/intel/text.py b/backend/src/fraudlens/intel/text.py index 72de261..5be3ad2 100644 --- a/backend/src/fraudlens/intel/text.py +++ b/backend/src/fraudlens/intel/text.py @@ -23,9 +23,9 @@ import joblib import numpy as np +from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression -from sklearn.multiclass import OneVsRestClassifier from sklearn.pipeline import FeatureUnion, Pipeline from ..config import Settings @@ -67,10 +67,32 @@ def build_pipeline(seed: int = 7) -> Pipeline: # cues would outweigh them and the wording would stop counting. transformer_weights={"chars": 1.0, "cues": 0.5}, ) # fmt: skip - model = OneVsRestClassifier( - LogisticRegression(C=2.0, max_iter=3000, class_weight="balanced", random_state=seed) - ) - return Pipeline([("features", features), ("model", model)]) + return Pipeline([("features", features), ("model", MultiHead(C=2.0, seed=seed))]) + + +class MultiHead(BaseEstimator, ClassifierMixin): + """One logistic regression per output, like one-vs-rest, except that a label of + -1 means "not known" and leaves that row out of that output. A message from the + public corpus (`external.py`) is known to be a scam or not, but not of which + category, so it trains the scam output only.""" + + def __init__(self, C: float = 2.0, seed: int = 7) -> None: + self.C = C + self.seed = seed + + def fit(self, X, Y) -> MultiHead: + Y = np.asarray(Y) + self.estimators_ = [] + for j in range(Y.shape[1]): + known = Y[:, j] >= 0 + est = LogisticRegression( + C=self.C, max_iter=3000, class_weight="balanced", random_state=self.seed + ) + self.estimators_.append(est.fit(X[known], Y[known, j])) + return self + + def predict_proba(self, X) -> np.ndarray: + return np.column_stack([est.predict_proba(X)[:, 1] for est in self.estimators_]) @dataclass diff --git a/backend/src/fraudlens/intel/train.py b/backend/src/fraudlens/intel/train.py index 3753874..3c2ac26 100644 --- a/backend/src/fraudlens/intel/train.py +++ b/backend/src/fraudlens/intel/train.py @@ -2,14 +2,20 @@ python -m fraudlens.intel.train -Writes the model and `report.json` under `artifacts/intel/`. Two test sets are +Writes the model and `report.json` under `artifacts/intel/`. Three test sets are reported and they mean different things: - `test`: scripts the model trained on, in wordings it never saw. - `unseen`: whole scam scripts (and harmless topics) kept out of training. +- `external`: the test split of a public, human-labelled Bangla/Banglish/English + smishing corpus (`external.py`), never trained on. -Both are generated from templates written for this project. Neither is a -measurement on messages real customers received. +`test` and `unseen` are generated from templates written for this project. +`external` is the one set here that the author of the corpus did not write. The +public corpus's training split is trained on (for the scam-or-not output only) +when it can be fetched; offline, the model trains on the written corpus alone and +the report says so. None of the three is a measurement on messages real customers +of this service received. """ from __future__ import annotations @@ -20,15 +26,16 @@ from pathlib import Path import numpy as np -from sklearn.metrics import average_precision_score, roc_auc_score +from sklearn.metrics import average_precision_score, f1_score, roc_auc_score from ..config import Settings +from . import external from .analyze import check_message -from .corpus import Sample, generate, load_families +from .corpus import TYPOLOGIES, Sample, generate, load_families from .taxonomy import load_taxonomy from .text import SCAM, TextModel, build_pipeline, intel_dir, normalise -VERSION = "v1" +VERSION = "v2" REPORT_FILE = "report.json" # The share of harmless messages each level may flag, set on the validation split. CAUTION_FPR = 0.03 @@ -44,6 +51,9 @@ "The cue patterns were written by hand, by the author of the corpus, with the " "trained scripts in view. They were not tuned against the held-out scripts, but " "the same person wrote both, so real messages will match them less often.", + "The public corpus (`external`) has its own label noise, and about 4% of its test " + "texts repeat a training text once normalised; its numbers hold for messages like " + "it, not for every scam a customer will receive.", "It reads text only: no screenshots, no voice calls, no images of QR codes.", "Scripts that share no wording with the trained ones are caught less often; " "the `unseen` numbers are the estimate of that.", @@ -64,6 +74,27 @@ def _rate(flags: np.ndarray) -> float | None: return round(float(flags.mean()), 4) if len(flags) else None +def _macro_f1(y: np.ndarray, flagged: np.ndarray) -> float | None: + """Binary macro-F1 (the mean of the scam and the harmless F1), when both occur.""" + if len(set(y.tolist())) < 2: + return None + return round(float(f1_score(y, flagged.astype(int), average="macro")), 4) + + +def _caution_high(scores: np.ndarray, benign: np.ndarray, source: np.ndarray) -> dict: + """Thresholds that flag at most CAUTION_FPR / HIGH_FPR of each source's harmless + validation messages. The public harmless messages are mostly easy (promotions, + chat); pooled with ours they would let the threshold drift onto our hard ones.""" + out = {} + for name, fpr in (("caution", CAUTION_FPR), ("high", HIGH_FPR)): + out[name] = max( + _threshold(scores[benign & (source == src)], fpr) + for src in sorted(set(source[benign].tolist())) + ) + out["high"] = max(out["high"], out["caution"]) + return {k: round(v, 6) for k, v in out.items()} + + def _split_report( model: TextModel, samples: list[Sample], scores: np.ndarray, per_family: bool ) -> dict: @@ -83,6 +114,7 @@ def _split_report( "recall": _rate(flagged[scam]), "false_positive_rate": _rate(flagged[harmless]), "false_positive_rate_hard": _rate(flagged[hard]), + "macro_f1": _macro_f1(y[:, 0], flagged), } # Categories are named only for a message already flagged, so they are measured that way. @@ -113,9 +145,21 @@ def _split_report( "messages": int((langs == lang).sum()), "recall": _rate(flagged[scam & (langs == lang)]), "false_positive_rate": _rate(flagged[harmless & (langs == lang)]), + "macro_f1": _macro_f1(y[langs == lang, 0], flagged[langs == lang]), } for lang in sorted(set(langs)) } + # Each typology's scams against every harmless message of the same split. + typ = np.array([s.typology or "" for s in samples]) + out["typologies"] = { + t: { + "scam": int((typ == t).sum()), + "recall": _rate(flagged[typ == t]), + "macro_f1": _macro_f1(y[(typ == t) | harmless, 0], flagged[(typ == t) | harmless]), + } + for t in TYPOLOGIES + if (typ == t).any() + } if per_family: families = np.array([s.family for s in samples]) out["families"] = { @@ -129,8 +173,42 @@ def _split_report( return out -def train(directory: Path, seed: int = 7, per_template: int = 30) -> dict: +def _external_report(model: TextModel, items: list[external.External], scores: np.ndarray) -> dict: + y = np.array([e.scam for e in items], dtype=int) + langs = np.array([e.lang for e in items]) + out: dict = { + "messages": len(items), + "scam": int(y.sum()), + "harmless": int(len(y) - y.sum()), + "auc": round(float(roc_auc_score(y, scores)), 4), + "average_precision": round(float(average_precision_score(y, scores)), 4), + } + for name in ("caution", "high"): + flagged = scores >= model.thresholds[name] + out[name] = { + "recall": _rate(flagged[y == 1]), + "false_positive_rate": _rate(flagged[y == 0]), + "macro_f1": _macro_f1(y, flagged), + } + flagged = scores >= model.thresholds["caution"] + out["languages"] = { + lang: { + "messages": int((langs == lang).sum()), + "recall": _rate(flagged[(y == 1) & (langs == lang)]), + "false_positive_rate": _rate(flagged[(y == 0) & (langs == lang)]), + "macro_f1": _macro_f1(y[langs == lang], flagged[langs == lang]), + } + for lang in sorted(set(langs.tolist())) + } + return out + + +def train( + directory: Path, seed: int = 7, per_template: int = 30, settings: Settings | None = None +) -> dict: + """`settings` lets it fetch the public corpus; without it, the written corpus only.""" taxonomy = load_taxonomy() + public = external.ensure(settings) if settings is not None else None samples = generate(seed=seed, per_template=per_template) by_split: dict[str, list[Sample]] = {} for s in samples: @@ -138,26 +216,41 @@ def train(directory: Path, seed: int = 7, per_template: int = 30) -> dict: outputs = [SCAM, *taxonomy.ids] fit = by_split["train"] + texts = [s.text for s in fit] + targets = _targets(fit, outputs) + if public: + # Public messages say scam or not, never which kind: -1 leaves them out of + # the category outputs (`text.MultiHead`). + texts += [e.text for e in public["train"]] + extra = np.full((len(public["train"]), len(outputs)), -1, dtype=int) + extra[:, 0] = [e.scam for e in public["train"]] + targets = np.vstack([targets, extra]) pipeline = build_pipeline(seed) - pipeline.fit([normalise(s.text) for s in fit], _targets(fit, outputs)) + pipeline.fit([normalise(t) for t in texts], targets) model = TextModel(VERSION, pipeline, outputs, {"category": CATEGORY_THRESHOLD}) val = by_split["val"] - val_scores = model.score([s.text for s in val]) - benign = val_scores[[not s.labels for s in val], 0] - model.thresholds["caution"] = round(_threshold(benign, CAUTION_FPR), 6) - model.thresholds["high"] = round( - max(_threshold(benign, HIGH_FPR), model.thresholds["caution"]), 6 - ) + val_texts = [s.text for s in val] + benign = [not s.labels for s in val] + source = ["corpus"] * len(val) + if public: + val_texts += [e.text for e in public["val"]] + benign += [not e.scam for e in public["val"]] + source += ["external"] * len(public["val"]) + val_scores = model.score(val_texts)[:, 0] + model.thresholds.update(_caution_high(val_scores, np.array(benign), np.array(source))) families = load_families() report = { "version": VERSION, "taxonomy_version": taxonomy.version, "seed": seed, - "model": "one-vs-rest logistic regression on character n-grams (2-5) and cues", + "model": "one logistic regression per output on character n-grams (2-5) and cues", "thresholds": model.thresholds, - "threshold_targets": {"caution_fpr": CAUTION_FPR, "high_fpr": HIGH_FPR, "set_on": "val"}, + "threshold_targets": { + "caution_fpr": CAUTION_FPR, "high_fpr": HIGH_FPR, + "set_on": "val: each source's harmless messages, the stricter threshold", + }, "corpus": { "families": len(families), "scam_families": sum(bool(f.labels) for f in families), @@ -166,12 +259,21 @@ def train(directory: Path, seed: int = 7, per_template: int = 30) -> dict: "messages": dict(Counter(s.split for s in samples)), "languages": dict(Counter(s.lang for s in samples)), }, + "public_corpus": external.provenance() | { + "messages": {split: len(items) for split, items in public.items()}, + "used_for": "train: scam output only; val: thresholds; test: reported only", + } if public else None, "test": _split_report(model, by_split["test"], model.score( [s.text for s in by_split["test"]]), per_family=False), "unseen": _split_report(model, by_split["unseen"], model.score( [s.text for s in by_split["unseen"]]), per_family=True), "limits": LIMITS, } # fmt: skip + if public: + test = public["test"] + report["external"] = _external_report( + model, test, model.score([e.text for e in test])[:, 0] + ) model.save(directory, taxonomy_version=taxonomy.version, seed=seed) (directory / REPORT_FILE).write_text(json.dumps(report, indent=2)) return report @@ -181,17 +283,22 @@ def main() -> None: parser = argparse.ArgumentParser(description="Train the scam-message classifier") parser.add_argument("--seed", type=int, default=7) args = parser.parse_args() - report = train(intel_dir(Settings()), seed=args.seed) - for split in ("test", "unseen"): + settings = Settings() + report = train(intel_dir(settings), seed=args.seed, settings=settings) + if report["public_corpus"] is None: + print("public corpus not available: trained on the written corpus only") + for split in ("test", "unseen", "external"): + if split not in report: + continue r = report[split] print( - f"{split:7} auc {r['auc']:.3f} " + f"{split:8} auc {r['auc']:.3f} " f"caution: recall {r['caution']['recall']:.3f} fpr " f"{r['caution']['false_positive_rate']:.3f} " - f"(hard {r['caution']['false_positive_rate_hard']}) " + f"macro-f1 {r['caution']['macro_f1']:.3f} " f"high: recall {r['high']['recall']:.3f} fpr {r['high']['false_positive_rate']:.3f}" ) - print(f"written to {intel_dir(Settings())}") + print(f"written to {intel_dir(settings)}") if __name__ == "__main__": diff --git a/backend/src/fraudlens/mlops/feedback.py b/backend/src/fraudlens/mlops/feedback.py index f50ddbf..ebe5327 100644 --- a/backend/src/fraudlens/mlops/feedback.py +++ b/backend/src/fraudlens/mlops/feedback.py @@ -8,49 +8,69 @@ Only what was alerted can be reviewed, so these labels are a biased sample: they say where the model was right or wrong among its alerts and nothing about the fraud it let through. They are added to the training data, never used in its place. + +A customer appeal a reviewer approved ("this is a genuine payment", checked by a +person) labels that one payment `legitimate`, unless its case reached a verdict: +the case looked at more and outranks it. A rejected appeal labels nothing. """ from __future__ import annotations import numpy as np import pandas as pd -from sqlalchemy import func, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session from ..features import FEATURES -from ..platform.models import Case, Decision, Transaction +from ..platform.models import Appeal, Case, Decision, Transaction LABELS = {"confirmed_fraud": 1, "legitimate": 0} +# An approved appeal whose payment has no case verdict to outrank it. +_APPEAL_ONLY = and_( + Appeal.status == "approved", + or_(Decision.case_id.is_(None), Case.verdict.is_(None), Case.verdict.not_in(LABELS)), +) + def labels(s: Session) -> pd.DataFrame: - """One row per reviewed alert, in `models.train.FEEDBACK_COLUMNS` plus case and verdict.""" - rows = s.execute( - select( - Decision.txn_id, Transaction.ts, Transaction.type, - Case.id, Case.verdict, Decision.features, - ) + """One row per reviewed alert, in `models.train.FEEDBACK_COLUMNS` plus case, verdict + and where the label came from (`case` or `appeal`).""" + columns = (Decision.txn_id, Transaction.ts, Transaction.type, Case.id, Decision.features) + reviewed = s.execute( + select(*columns, Case.verdict) .join(Transaction, Transaction.txn_id == Decision.txn_id) .join(Case, Case.id == Decision.case_id) .where(Case.verdict.in_(LABELS)) - .order_by(Decision.txn_id) - ).all() # fmt: skip + ).all() + appealed = s.execute( + select(*columns) + .join(Transaction, Transaction.txn_id == Decision.txn_id) + .join(Appeal, Appeal.txn_id == Decision.txn_id) + .outerjoin(Case, Case.id == Decision.case_id) + .where(_APPEAL_ONLY) + ).all() + rows = sorted( + [(*row, "case") for row in reviewed] + [(*row, "legitimate", "appeal") for row in appealed], + key=lambda row: row[0], + ) # A decision made by code with another feature list cannot be trained on. - rows = [row for row in rows if len(row.features) == len(FEATURES)] + rows = [row for row in rows if len(row[4]) == len(FEATURES)] frame = pd.DataFrame( { - "txn_id": np.array([row.txn_id for row in rows], dtype=np.int64), + "txn_id": np.array([row[0] for row in rows], dtype=np.int64), # Naive UTC, as in the dataset the training frame is read from. - "ts": pd.to_datetime([row.ts for row in rows], utc=True).tz_localize(None), - "type": [row.type for row in rows], - "case_id": [row.id for row in rows], - "verdict": [row.verdict for row in rows], + "ts": pd.to_datetime([row[1] for row in rows], utc=True).tz_localize(None), + "type": [row[2] for row in rows], + "case_id": pd.array([row[3] for row in rows], dtype="Int64"), + "verdict": [row[5] for row in rows], + "source": [row[6] for row in rows], } ) frame["y"] = frame["verdict"].map(LABELS).astype("int8") # A confirmed payment into a wallet is what the mule model learns from. frame["y_mule"] = (frame["y"] * (frame["type"] == "SEND_MONEY")).astype("int8") - features = np.array([row.features for row in rows], dtype=np.float64).reshape(-1, len(FEATURES)) + features = np.array([row[4] for row in rows], dtype=np.float64).reshape(-1, len(FEATURES)) return pd.concat([frame, pd.DataFrame(features, columns=list(FEATURES))], axis=1) @@ -71,11 +91,19 @@ def summary(s: Session) -> dict: .group_by(Case.verdict) ).all() ) + appeals = s.scalar( + select(func.count()) + .select_from(Appeal) + .join(Decision, Decision.txn_id == Appeal.txn_id) + .outerjoin(Case, Case.id == Decision.case_id) + .where(_APPEAL_ONLY) + ) return { "cases": {verdict: cases.get(verdict, 0) for verdict in (*LABELS, "inconclusive")}, "labels": { "fraud": alerts.get("confirmed_fraud", 0), - "legitimate": alerts.get("legitimate", 0), + "legitimate": alerts.get("legitimate", 0) + appeals, }, + "approved_appeals": appeals, "last_verdict_at": s.scalar(select(func.max(Case.closed_at))), } diff --git a/backend/src/fraudlens/models/adversary.py b/backend/src/fraudlens/models/adversary.py new file mode 100644 index 0000000..11a2cb6 --- /dev/null +++ b/backend/src/fraudlens/models/adversary.py @@ -0,0 +1,469 @@ +"""Adaptive-adversary experiment: does detection survive scammers who learn? + + uv run python -m fraudlens.models.adversary # backend/data/full and the served model + +The default world runs unchanged to its last day (checked against the stored +dataset, so the stored feature-engine snapshot can carry on from it). Then the +served model is "deployed", new scam cells set up for a week, and the adversary +runs in rounds (`simulator.adversary`): after each round it moves its tactic mix +toward whatever got past the model deployed in that arm. Four arms, each its own +simulation because the adversary reacts to its own arm's model: + +- control: frozen model, adversary does not adapt (the round-0 mix throughout) +- frozen: the served model, never changed +- retrain: retrained after every round, by the `models.train` recipe with the + labels `mlops.retrain` would have (see `labels`) +- drift_gated: retrained only after a round in which `mlops.drift.measure` raises + an alert on the served score + +Writes `artifacts/reports/adversary.json`. Nothing in the model registry or the +dataset is written. +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import asdict, replace +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.ensemble import IsolationForest +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import StratifiedKFold, cross_val_predict + +from ..config import settings +from ..decision.insights import drift_reference, reference_bins +from ..features import FEATURES, RECIPIENT_FEATURES, FeatureEngine +from ..features.build import LABEL_COLUMNS, Replayer, epoch_seconds +from ..mlops import drift +from ..simulator.adversary import TACTICS, Adversary, AdversaryConfig, deploy +from ..simulator.config import DAY, SimConfig +from ..simulator.engine import K_WALLET, Simulation +from ..simulator.generate import DISTRICT_NAMES, _when, party_id, transaction_table +from . import metrics, registry +from .bundle import ANOMALY_FEATURES, ModelBundle, logit +from .data import AMBIGUOUS_ROLES, LOSS_ROLES, load_frame +from .train import FEEDBACK_COLUMNS, FUSION_MIN_GAIN, SEED, choose_thresholds, fit_booster + +ARMS = ("control", "frozen", "retrain", "drift_gated") +REPORT_FILE = "adversary.json" +TIER = "warn" # the operating point the card's headline case recall is quoted at +SHAP_TOP = 8 + + +# ----------------------------------------------------------------- the world + + +def sim_config(data_dir: Path) -> SimConfig: + """The configuration the stored dataset was generated with.""" + cfg = json.loads((data_dir / "meta.json").read_text())["config"] + return SimConfig(**{**cfg, "start": datetime.fromisoformat(cfg["start"])}) + + +class World: + """One arm's running simulation and the feature engine that follows it.""" + + def __init__(self, data_dir: Path, adversary: Adversary) -> None: + self.cfg = cfg = sim_config(data_dir) + self.sim = sim = Simulation(cfg) + sim.run() + stored = pd.read_parquet(data_dir / "transactions.parquet", columns=["amount"])["amount"] + if len(stored) != len(sim.cols["amount"]) or not np.isclose( + stored.sum(), float(np.sum(sim.cols["amount"])) + ): + raise RuntimeError( + f"the simulator no longer reproduces {data_dir}: regenerate the dataset " + "(make data features) before running the adversary experiment" + ) + self.engine = FeatureEngine.load(data_dir / "engine_after_test.pkl") + self.known_wallets = sim.w.n_wallets + self.done = len(sim.cols["ts"]) + self.day = cfg.days + self.planner = deploy(sim, adversary, cfg.days) + + def step(self, days: int) -> pd.DataFrame: + """Simulate `days` more days and replay them: one row per scored transaction.""" + sim, cfg, engine = self.sim, self.cfg, self.engine + sim.run_days(self.day, self.day + days) + self.day += days + end_ts = self.day * DAY + txns = transaction_table(sim, self.done) + self.done += len(txns) + + w = sim.w + new = range(self.known_wallets, w.n_wallets) + created = epoch_seconds(_when(cfg, [w.created_ts[i] for i in new])).tolist() + for i, ts in zip(new, created, strict=True): + engine.register_wallet(party_id(K_WALLET, i), ts, DISTRICT_NAMES[w.district[i]]) + self.known_wallets = w.n_wallets + + flagged = sorted( + (m.flagged_ts, party_id(K_WALLET, m.wallet)) + for m in self.planner.mules.values() + if m.flagged_ts < end_ts and party_id(K_WALLET, m.wallet) not in engine.flagged + ) + flags = pd.DataFrame( + { + "wallet_id": [f[1] for f in flagged], + "flagged_at": _when(cfg, [f[0] for f in flagged]), + } + ) + pos, x = Replayer(engine, flags).run(txns) + frame = txns.iloc[pos][list(LABEL_COLUMNS)].reset_index(drop=True) + frame = pd.concat([frame, pd.DataFrame(x, columns=list(FEATURES))], axis=1) + frame["y"] = frame["is_fraud"].astype("int8") + frame["y_loss"] = frame["fraud_role"].isin(LOSS_ROLES).astype("int8") + frame = frame[~frame["fraud_role"].isin(AMBIGUOUS_ROLES)].reset_index(drop=True) + mules = self.planner.mules + first = {party_id(K_WALLET, k): m.first_fraud_ts for k, m in mules.items()} + first_ts = _when(cfg, frame["receiver_id"].map(first).fillna(np.inf).to_numpy()) + frame["y_mule"] = ((frame["type"] == "SEND_MONEY") & (frame["ts"] >= first_ts)).astype( + "int8" + ) + frame["tactic"] = frame["case_id"].map(self.planner.tactic).fillna("") + return frame + + def reported_cases(self) -> set[int]: + """Scams a victim has reported by now: what a complaint desk would know.""" + now = self.day * DAY + return {case for case, ts in self.planner.reported.items() if ts < now} + + +# ---------------------------------------------------------------- the models + + +class Deployed: + """A model as served: bundle, tier thresholds and its drift reference.""" + + def __init__(self, bundle: ModelBundle, thresholds: dict[str, float], reference: dict): + self.bundle = bundle + self.thresholds = thresholds + self.reference = reference + + def risk(self, frame: pd.DataFrame) -> np.ndarray: + return self.bundle.score(frame[list(FEATURES)].to_numpy(dtype=np.float64))["risk"] + + +def fit_bundle(fold: dict[str, pd.DataFrame], version: str) -> tuple[ModelBundle, np.ndarray]: + """The `models.train.train` recipe without its reports: boosted transaction and + mule models, isolation forest, fusion and calibration on val_b, thresholds on val_b. + Returns the bundle (thresholds in its manifest) and its calibrated val_b risk.""" + cols, recipient = list(FEATURES), list(RECIPIENT_FEATURES) + send = {name: part[part["type"] == "SEND_MONEY"] for name, part in fold.items()} + txn = fit_booster(fold["train"], fold["val_a"], cols, "y") + mule = fit_booster(send["train"], send["val_a"], recipient, "y_mule") + behaviour = fold["train"][list(ANOMALY_FEATURES)] + sample = behaviour.fillna(behaviour.median()).sample( + min(len(behaviour), 200_000), random_state=SEED + ) + anomaly = IsolationForest(n_estimators=100, random_state=SEED, n_jobs=-1).fit(sample.to_numpy()) + bundle = ModelBundle( + version=version, txn=txn, mule=mule, anomaly=anomaly, + anomaly_fill=behaviour.median().to_numpy(), fusion_coef=np.zeros(4), + fusion_intercept=0.0, calibration=(1.0, 0.0), risk_source="txn", manifest={}, + ) # fmt: skip + vb = fold["val_b"] + parts = bundle.components(vb[cols].to_numpy()) + z, y = bundle.fusion_inputs(parts), vb["y"].to_numpy() + fusion = LogisticRegression(C=1.0, max_iter=2000) + cv = StratifiedKFold(5, shuffle=True, random_state=SEED) + fused_cv = cross_val_predict(fusion, z, y, cv=cv, method="predict_proba")[:, 1] + fusion.fit(z, y) + bundle.fusion_coef, bundle.fusion_intercept = fusion.coef_[0], float(fusion.intercept_[0]) + val_txn = metrics.ranking(y, parts["txn"])["pr_auc"] + val_fused = metrics.ranking(y, fused_cv)["pr_auc"] + bundle.risk_source = "fused" if val_fused >= val_txn + FUSION_MIN_GAIN else "txn" + raw = bundle.raw_risk(parts) + platt = LogisticRegression(C=1e6, max_iter=2000).fit(logit(raw).reshape(-1, 1), y) + bundle.calibration = (float(platt.coef_[0][0]), float(platt.intercept_[0])) + risk = bundle.calibrate(raw) + bundle.manifest = {"version": version, "thresholds": choose_thresholds(y, risk)} + return bundle, risk + + +class Lab: + """What every arm shares: the training folds, drift references, the served model.""" + + def __init__(self, data_dir: Path, models_root: Path | None, version: str | None) -> None: + self.data_dir = data_dir + frame = load_frame(data_dir) + clean = frame[~frame["ambiguous"]] + self.fold = {n: clean[clean["fold"] == n] for n in ("train", "val_a", "val_b", "test")} + served = registry.load(version, models_root) + self.version = served.version + thresholds = dict(served.manifest["thresholds"]) + ref_rows = pd.concat([self.fold["train"], self.fold["val_b"]], ignore_index=True) + risk = served.score(ref_rows[list(FEATURES)].to_numpy(dtype=np.float64))["risk"] + self.population_reference = drift_reference(ref_rows, risk, thresholds) + self.served = Deployed(served, thresholds, self.population_reference) + + test = self.fold["test"] + test_risk = self.served.risk(test) + self.served_check = { + "test_pr_auc": metrics.ranking(test["y"].to_numpy(), test_risk)["pr_auc"], + "manifest_pr_auc": served.manifest.get("headline", {}).get("pr_auc"), + } + + # The same monitor pointed at confirmed fraud only: training-period fraud as + # the feature reference, the model's scores on val_b fraud as the score reference. + fraud_train = self.fold["train"][self.fold["train"]["y"] == 1] + self.fraud_features = { + name: reference_bins(fraud_train[name].to_numpy(dtype=np.float64)) for name in FEATURES + } + self.val_b_fraud = self.fold["val_b"][self.fold["val_b"]["y"] == 1] + + def fraud_reference(self, model: Deployed) -> dict: + val_b_fraud_risk = model.risk(self.val_b_fraud) + thresholds = model.thresholds + return { + "features": self.fraud_features, + "score": reference_bins(val_b_fraud_risk), + "validation_alert_rate": round(float((val_b_fraud_risk >= thresholds[TIER]).mean()), 5), + "validation_hold_rate": round( + float((val_b_fraud_risk >= thresholds["hold"]).mean()), 5 + ), + } + + def retrain(self, feedback: pd.DataFrame, version: str) -> Deployed: + """Verdicts and complaints join the training fold, as `models.train.add_feedback` does.""" + fold = dict(self.fold) + rows = feedback[list(FEEDBACK_COLUMNS)].astype({"y": "int8", "y_mule": "int8"}) + fold["train"] = pd.concat([fold["train"], rows], ignore_index=True) + bundle, val_b_risk = fit_bundle(fold, version) + thresholds = bundle.manifest["thresholds"] + reference = { + **self.population_reference, + "score": reference_bins(val_b_risk), + "validation_alert_rate": round(float((val_b_risk >= thresholds[TIER]).mean()), 5), + "validation_hold_rate": round(float((val_b_risk >= thresholds["hold"]).mean()), 5), + } + return Deployed(bundle, thresholds, reference) + + +# -------------------------------------------------------------- measurement + + +def labels(frames: list[pd.DataFrame], alerted: list[np.ndarray], reported: set[int]): + """What a retrain could learn from by now, in `FEEDBACK_COLUMNS`. + + Verdicts: every alerted transaction, closed with its true label (as `make review` + closes cases with the simulation's ground truth). Complaints: every fraud-chain + transaction of a scam whose victim has reported it. Fraud nobody alerted and + nobody reported stays unlabelled, as it would in production. + """ + parts = [] + for frame, hit in zip(frames, alerted, strict=True): + complaint = frame["case_id"].isin(reported).to_numpy() & (frame["y"] == 1).to_numpy() + parts.append(frame[hit | complaint]) + rows = pd.concat(parts, ignore_index=True).drop_duplicates("txn_id") + rows = rows.assign(y_mule=(rows["y"] * (rows["type"] == "SEND_MONEY")).astype("int8")) + return rows + + +def outcomes(frame: pd.DataFrame, risk: np.ndarray, threshold: float) -> dict: + """Recall and precision at the tier threshold, overall and by tactic.""" + hit = risk >= threshold + overall = metrics.alert_outcomes(frame, hit) + by_tactic = metrics.alert_outcomes(frame.assign(typology=frame["tactic"]), hit)["by_typology"] + keep = ("loss_txns", "cases", "loss_txn_recall", "case_recall", "taka_recall", + "taka_recall_with_exit_holds") # fmt: skip + return { + "pr_auc": metrics.ranking(frame["y"].to_numpy(), risk)["pr_auc"], + **{k: overall[k] for k in ("alerts_per_day", "alert_rate", "precision", *keep)}, + "by_tactic": {t: {k: v[k] for k in keep} for t, v in by_tactic.items() if t}, + } + + +def tactic_results(frame: pd.DataFrame, risk: np.ndarray, threshold: float): + """Per tactic: (scams run, scams with no victim transfer alerted) — the adversary's view.""" + victim = frame[frame["y_loss"] == 1].assign( + hit=risk[(frame["y_loss"] == 1).to_numpy()] >= threshold + ) + caught = victim.groupby("case_id").agg(tactic=("tactic", "first"), hit=("hit", "any")) + out = {} + for tactic, rows in caught.groupby("tactic"): + out[str(tactic)] = (int(len(rows)), int((~rows["hit"]).sum())) + return out + + +def monitor(reference: dict, frame: pd.DataFrame, risk: np.ndarray, thresholds: dict) -> dict: + """`mlops.drift.measure`, summarised: does it raise an alert?""" + if len(frame) < drift.MIN_ROWS: + return {"status": "not_enough_data", "rows": int(len(frame)), "needed": drift.MIN_ROWS} + m = drift.measure(reference, frame[list(FEATURES)].to_numpy(dtype=np.float64), risk, thresholds) + return { + "rows": m["rows"], + "score_psi": m["score"]["psi"], + "score_status": m["score"]["status"], + "alert": m["score"]["status"] != "stable", + "alert_rate": m["score"]["alert_rate"], + "validation_alert_rate": m["score"]["validation_alert_rate"], + "feature_status": m["feature_status"], + "most_shifted": [{"feature": f["feature"], "psi": f["psi"]} for f in m["features"][:5]], + } + + +def shap_shares(model: Deployed, frame: pd.DataFrame) -> list[dict]: + """Mean |SHAP| of the transaction model on victim transfers, as shares of the total.""" + victims = frame[frame["y_loss"] == 1] + if victims.empty: + return [] + txn, _ = model.bundle.contributions(victims[list(FEATURES)].to_numpy(dtype=np.float64)) + mean = np.abs(txn).mean(axis=0) + order = np.argsort(-mean)[:SHAP_TOP] + total = mean.sum() or 1.0 + return [{"feature": FEATURES[i], "share": round(float(mean[i] / total), 4)} for i in order] + + +# ------------------------------------------------------------------- arms + + +def run_arm(arm: str, lab: Lab, acfg: AdversaryConfig, log=print) -> dict: + adversary = Adversary(replace(acfg, adapt=arm != "control")) + world = World(lab.data_dir, adversary) + world.step(acfg.warmup_days) # the new cells open and warm up their mules + model, model_name = lab.served, lab.version + frames, alerted, rounds = [], [], [] + for r in range(acfg.rounds): + mix = {t: round(v, 4) for t, v in adversary.mix.items()} + frame = world.step(acfg.round_days) + risk = model.risk(frame) + threshold = model.thresholds[TIER] + frames.append(frame.assign(risk=risk)) + alerted.append(risk >= threshold) + + result = outcomes(frame, risk, threshold) + population = monitor(model.reference, frame, risk, model.thresholds) + known = labels(frames, alerted, world.reported_cases()) + known_now = known[known["txn_id"].isin(frame["txn_id"]) & (known["y"] == 1)] + fraud_seg = monitor( + lab.fraud_reference(model), known_now, + model.risk(known_now) if len(known_now) else np.array([]), model.thresholds, + ) # fmt: skip + tactics = tactic_results(frame, risk, threshold) + new_mix = adversary.update(tactics) + + retrain = arm == "retrain" or (arm == "drift_gated" and population.get("alert", False)) + rounds.append( + { + "round": r, + "days": [world.day - acfg.round_days, world.day], + "model": model_name, + "mix": mix, + "scams_by_tactic": { + t: {"run": n, "got_through": g} for t, (n, g) in tactics.items() + }, + **result, + "drift_population": population, + "drift_confirmed_fraud": fraud_seg, + "labels_known": {"rows": int(len(known)), "fraud": int(known["y"].sum())}, + "retrain_after": bool(retrain and r < acfg.rounds - 1), + } + ) + log( + f" {arm} round {r}: case recall {result['case_recall']}, victim-txn recall " + f"{result['loss_txn_recall']}, score PSI {population.get('score_psi')}, " + f"model {model_name}" + ) + if retrain and r < acfg.rounds - 1: + model_name = f"{arm}-r{r + 1}" + model = lab.retrain(known, model_name) + rounds[-1]["mix_after"] = {t: round(v, 4) for t, v in new_mix.items()} + + pooled = pd.concat(frames, ignore_index=True) + risk_all = pooled["risk"].to_numpy() + by_tactic = metrics.alert_outcomes( + pooled.assign(typology=pooled["tactic"]), np.concatenate(alerted) + )["by_typology"] + return { + "rounds": rounds, + "pooled_pr_auc": metrics.ranking(pooled["y"].to_numpy(), risk_all)["pr_auc"], + "pooled_by_tactic": { + t: { + k: v[k] + for k in ( + "loss_txns", + "cases", + "loss_txn_recall", + "case_recall", + "taka_recall_with_exit_holds", + ) + } # fmt: skip + for t, v in by_tactic.items() + if t + }, + "shap_first_round": shap_shares(lab.served, frames[0]), + "shap_last_round": shap_shares(model, frames[-1]), + "last_model": model_name, + } + + +def run( + data_dir: Path, + models_root: Path | None = None, + version: str | None = None, + acfg: AdversaryConfig | None = None, + arms: tuple[str, ...] = ARMS, + log=print, +) -> dict: + t0 = time.perf_counter() + acfg = acfg or AdversaryConfig() + lab = Lab(data_dir, models_root, version) + log(f"served model {lab.version}: test PR-AUC {lab.served_check['test_pr_auc']}") + results = {} + for arm in arms: + t = time.perf_counter() + results[arm] = run_arm(arm, lab, acfg, log) + log(f"{arm} done in {time.perf_counter() - t:.0f}s") + summary = { + arm: { + "case_recall": [r["case_recall"] for r in res["rounds"]], + "loss_txn_recall": [r["loss_txn_recall"] for r in res["rounds"]], + "drift_alert": [r["drift_population"].get("alert") for r in res["rounds"]], + } + for arm, res in results.items() + } + return { + "generated_at": datetime.now(UTC).isoformat(timespec="seconds"), + "synthetic": True, + "data_dir": str(data_dir), + "served_model": lab.version, + "served_model_check": lab.served_check, + "tier": TIER, + "tactics": list(TACTICS), + "adversary": asdict(acfg), + "sampling": "none: every transaction of every round is replayed and scored", + "arms": results, + "summary": summary, + "seconds": round(time.perf_counter() - t0, 1), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Adaptive-adversary experiment") + parser.add_argument("--data", type=Path, default=None) + parser.add_argument("--version", default=None, help="model to deploy (default: the served one)") + parser.add_argument("--rounds", type=int, default=AdversaryConfig.rounds) + parser.add_argument("--round-days", type=int, default=AdversaryConfig.round_days) + parser.add_argument("--cells", type=int, default=AdversaryConfig.n_cells) + parser.add_argument("--arms", default=",".join(ARMS)) + parser.add_argument("--out", type=Path, default=None) + args = parser.parse_args() + arms = tuple(a for a in args.arms.split(",") if a) + if unknown := set(arms) - set(ARMS): + parser.error(f"unknown arms: {sorted(unknown)}") + acfg = AdversaryConfig(rounds=args.rounds, round_days=args.round_days, n_cells=args.cells) + report = run(args.data or settings.dataset_dir, settings.models_dir, args.version, acfg, arms) + out = args.out or settings.artifacts_dir / "reports" / REPORT_FILE + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, default=str)) + print(json.dumps(report["summary"], indent=2)) + print(f"wrote {out} in {report['seconds']}s") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/models/bootstrap.py b/backend/src/fraudlens/models/bootstrap.py new file mode 100644 index 0000000..8865f96 --- /dev/null +++ b/backend/src/fraudlens/models/bootstrap.py @@ -0,0 +1,198 @@ +"""Bootstrap 95% confidence intervals for the headline test-period metrics. + +The headline numbers (scams caught at the warn tier, the false-alert rate on +legitimate payments, PR-AUC) are point estimates from one test period of 25 days +and 360 scams. This puts an interval around each, two ways: + +- by day: whole days are resampled. Days are the coarsest unit the data has, so + this keeps the correlation between transactions of the same day and the same scam. +- by scam: each scam (all its fraud-chain transactions) is one unit, and each + legitimate transaction is one unit. Scam and legitimate units are resampled + separately, so every replicate has the same number of each. + +Both are percentile intervals. Nothing is refitted: the model, its calibration and +its thresholds are fixed, so the intervals describe the uncertainty of the test +measurement, not of training. + + uv run python -m fraudlens.models.bootstrap # served model, data/full +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.metrics import average_precision_score + +from ..config import settings +from . import registry +from .data import load_frame + +OUTPUT_FILE = "bootstrap_ci.json" +REPLICATES = 1000 +SEED = 7 + + +def point_metrics( + y: np.ndarray, + loss: np.ndarray, + case: np.ndarray, + risk: np.ndarray, + alert: np.ndarray, + w: np.ndarray | None = None, +) -> dict[str, float]: + """Scam recall, false-alert rate and PR-AUC, with optional row weights. + + A scam counts as caught when at least one of its victim transfers is alerted + (the model card's "scams caught"). With weights, a scam's weight is the weight + of its victim transfers (all equal under both resampling schemes). + """ + w = np.ones(len(y)) if w is None else w + legit = y == 0 + fpr = float((w * (legit & alert)).sum() / max((w * legit).sum(), 1e-12)) + victim = pd.DataFrame({"case": case[loss], "hit": alert[loss], "w": w[loss]}) + per_case = victim.groupby("case").agg(hit=("hit", "any"), w=("w", "first")) + recall = float((per_case["w"] * per_case["hit"]).sum() / max(per_case["w"].sum(), 1e-12)) + keep = w > 0 + pr_auc = float(average_precision_score(y[keep], risk[keep], sample_weight=w[keep])) + return {"scam_recall": recall, "false_alert_rate": fpr, "pr_auc": pr_auc} + + +def _unit_weights(units: np.ndarray, rng: np.random.Generator) -> np.ndarray: + """Row weights from resampling the distinct units with replacement.""" + codes, inverse = np.unique(units, return_inverse=True) + counts = np.bincount(rng.integers(0, len(codes), len(codes)), minlength=len(codes)) + return counts[inverse].astype(np.float64) + + +def bootstrap( + test: pd.DataFrame, + risk: np.ndarray, + alert: np.ndarray, + replicates: int = REPLICATES, + seed: int = SEED, +) -> dict: + """Point estimates and percentile intervals under both schemes. + + `test` needs y, y_loss, case_id (negative or missing for legitimate rows) and day. + """ + y = test["y"].to_numpy().astype(int) + loss = test["y_loss"].to_numpy() == 1 + case = test["case_id"].fillna(-1).to_numpy().astype(np.int64) + day = test["day"].to_numpy() + alert = np.asarray(alert, dtype=bool) + risk = np.asarray(risk, dtype=np.float64) + rng = np.random.default_rng(seed) + + # By scam: fraud rows grouped by scam, legitimate rows on their own; the two + # strata are drawn separately. + fraud_rows = case >= 0 + row_ids = np.arange(len(y)) + + draws: dict[str, list[dict]] = {"by_day": [], "by_scam": []} + for _ in range(replicates): + draws["by_day"].append(point_metrics(y, loss, case, risk, alert, _unit_weights(day, rng))) + w = np.zeros(len(y)) + w[fraud_rows] = _unit_weights(case[fraud_rows], rng) + w[~fraud_rows] = _unit_weights(row_ids[~fraud_rows], rng) + draws["by_scam"].append(point_metrics(y, loss, case, risk, alert, w)) + + point = point_metrics(y, loss, case, risk, alert) + out = { + "point": {k: round(v, 5) for k, v in point.items()}, + "n": { + "rows": int(len(y)), + "legitimate": int((y == 0).sum()), + "scams": int(len(np.unique(case[loss]))), + "days": int(len(np.unique(day))), + }, + } + for scheme, rows in draws.items(): + frame = pd.DataFrame(rows) + out[scheme] = { + metric: { + "low": round(float(frame[metric].quantile(0.025)), 5), + "high": round(float(frame[metric].quantile(0.975)), 5), + "std": round(float(frame[metric].std()), 5), + } + for metric in frame.columns + } + return out + + +def run( + data_dir: Path, + model: str | None = None, + models_root: Path | None = None, + replicates: int = REPLICATES, +) -> dict: + from ..decision.evaluate import add_context, decide_batch + from ..decision.policy import DEFAULT_POLICY, RANK, load_policy + + root = models_root or registry.models_dir() + version = model or registry.current_version(root) + model_dir = root / version + manifest = json.loads((model_dir / "manifest.json").read_text()) + scores = pd.read_parquet(model_dir / "test_scores.parquet") + + frame = load_frame(data_dir) + test = frame[~frame["ambiguous"] & (frame["fold"] == "test")].reset_index(drop=True) + if not (scores["txn_id"].to_numpy() == test["txn_id"].to_numpy()).all(): + raise ValueError(f"{model_dir / 'test_scores.parquet'} does not match the test rows") + risk = scores["risk"].to_numpy() + threshold = float(manifest["thresholds"]["warn"]) + + # The served decision: the policy's tiers (model thresholds plus its rules). The + # headline false-alert rate comes from insights.json, so use the policy it used. + insights = model_dir / "insights.json" + policy_version = ( + json.loads(insights.read_text())["policy_version"] if insights.exists() else DEFAULT_POLICY + ) + policy = load_policy(policy_version) + bundle = registry.load(version, root) + context = add_context(test, pd.read_parquet(data_dir / "wallet_flags.parquet")) + outcomes, _ = decide_batch(policy, context, bundle) + policy_alert = np.array([RANK[o.tier] > 0 for o in outcomes]) + + report = { + "model_version": version, + "data_dir": str(data_dir), + "replicates": replicates, + "seed": SEED, + "interval": "percentile, 2.5% to 97.5%", + "model_warn_threshold": { + "threshold": threshold, + **bootstrap(test, risk, risk >= threshold, replicates), + }, + "policy_warn_or_above": { + "policy": policy.version, + **bootstrap(test, risk, policy_alert, replicates), + }, + } + (model_dir / OUTPUT_FILE).write_text(json.dumps(report, indent=2)) + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--data", type=Path, default=settings.data_dir / "full") + parser.add_argument("--model", default=None, help="version, default: the served one") + parser.add_argument("--replicates", type=int, default=REPLICATES) + args = parser.parse_args() + report = run(args.data, args.model, replicates=args.replicates) + for name in ("model_warn_threshold", "policy_warn_or_above"): + part = report[name] + print(f"\n{name} ({report['model_version']}, {report['replicates']} replicates)") + for metric, value in part["point"].items(): + ci = ", ".join( + f"{s} [{part[s][metric]['low']:.4f}, {part[s][metric]['high']:.4f}]" + for s in ("by_day", "by_scam") + ) + print(f" {metric}: {value:.4f} {ci}") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/models/label_realism.py b/backend/src/fraudlens/models/label_realism.py new file mode 100644 index 0000000..a53ff6d --- /dev/null +++ b/backend/src/fraudlens/models/label_realism.py @@ -0,0 +1,497 @@ +"""Label realism: what training only on reported scams costs, and what wins it back. + +uv run python -m fraudlens.models.label_realism # backend/data/full, 5 seeds +uv run python -m fraudlens.models.label_realism --seeds 7 11 # fewer seeds + +The served models are trained on the simulator's ground truth. A real wallet only +learns about the scams its victims report. This experiment trains the transaction +model the way `train.fit_booster` does (same parameters, train fold, early stopping +on val_a) under several label regimes and scores every one on the same ground-truth +test period, at the false-positive rate the served model runs at. + +Regimes: + +- ground_truth: every fraud-chain transaction labelled (what the served model had). +- reported_uniform: a case is labelled only if its victim reported it, with the + simulator's own process (50%, independent of anything). +- reported: the same 50% on average, but larger losses, urban victims, long-held + accounts and unauthorised takeovers report more, shame-heavy scams less. +- reported_pu: reported, then positive-unlabelled learning (Elkan & Noto 2008): + each unlabelled transaction counts as fraud with the weight its propensity implies. +- reported_propagated: reported, then soft labels through the mule network: other + money into or out of a wallet a reported scam paid, near the reported transfers. +- reported_feedback: reported, then the analyst-verdict loop: a model trained on + earlier reports alerts on the last training weeks, analysts label the alerts, and + the verdicts join the training data through `train.add_feedback`, the path + `mlops/feedback.py` feeds in production. +- reported_propagated_feedback: both. + +A reported case labels every transaction tagged with it (the victim's transfers and +the mule's forwards and cash-outs), as an investigation following the money would. +Reports arrive after a delay; only those in by the start of the test period, when +the model is trained, count. Unreported fraud is labelled clean, as it would be. +""" + +from __future__ import annotations + +import argparse +import json +import time +from datetime import UTC, datetime +from pathlib import Path + +import lightgbm as lgb +import numpy as np +import pandas as pd +from sklearn.model_selection import StratifiedKFold + +from ..config import settings +from ..features import FEATURES +from . import metrics, registry +from .data import load_frame +from .train import FEEDBACK_COLUMNS, LGB_PARAMS, add_feedback + +REPORT_FILE = "label_realism.json" +SEEDS = (7, 11, 13, 17, 19) +DAY = pd.Timedelta(days=1) + +# Report propensity, as log-odds terms. Synthetic assumptions, not measurements: the +# intercept is solved so that the average matches the simulator's `report_rate`. +# The simulator has no person age, so account tenure stands in for an older customer. +REPORT_LOGIT = {"log_loss": 0.6, "urban": 0.5, "tenure_1y": 0.4} +TYPOLOGY_LOGIT = { + "account_takeover": 1.0, # unauthorised: the victim disputes it with the provider + "impersonation": 0.0, + "wrong_send": 0.0, + "lottery_fee": -0.5, # victims who paid a "fee" are embarrassed to report + "investment_scam": -0.8, # and those who believed in the returns report late, if at all +} +# Report delay, as in the simulator: lognormal around a median, plus a few hours. +DELAY_MEDIAN_DAYS = {"investment_scam": 5.0} +DELAY_MEDIAN_DEFAULT_DAYS = 1.5 + +PROPAGATION_WEIGHT = 0.7 # fraud weight of a transfer that touched a reported mule +PROPAGATION_WINDOW = 3 * DAY # around the wallet's first and last reported transfer +PU_FOLDS = 3 +REVIEW_BUDGET = 0.01 # analysts review the top 1% of the review window, the headline budget +SERVED_TIER = "warn" # the operating point compared at: its false-positive rate on test +REGIMES = ( + "ground_truth", + "reported_uniform", + "reported", + "reported_pu", + "reported_pu_known_rate", + "reported_propagated", + "reported_feedback", + "reported_propagated_feedback", +) + + +# ---------------------------------------------------------------- reporting + + +def case_table(data_dir: Path) -> pd.DataFrame: + """Cases with what their reporting depends on: loss, victim's area and tenure.""" + cases = pd.read_parquet(data_dir / "cases.parquet") + wallets = pd.read_parquet(data_dir / "wallets.parquet").set_index("wallet_id") + victim = wallets.reindex(cases["victim_id"]) + cases["urban"] = (victim["area_type"].to_numpy() == "urban").astype(float) + tenure = (cases["started_at"] - victim["created_at"].to_numpy()) / DAY + cases["tenure_days"] = tenure.to_numpy() + return cases + + +def report_probability(cases: pd.DataFrame, rate: float, biased: bool = True) -> np.ndarray: + """Probability that a case is reported, averaging `rate` over the cases.""" + if not biased: + return np.full(len(cases), rate) + z = ( + REPORT_LOGIT["log_loss"] * np.log(cases["loss"].clip(lower=1.0) / cases["loss"].median()) + + REPORT_LOGIT["urban"] * cases["urban"].to_numpy() + + REPORT_LOGIT["tenure_1y"] * (cases["tenure_days"].to_numpy() >= 365) + + cases["typology"].map(TYPOLOGY_LOGIT).fillna(0.0).to_numpy() + ) + lo, hi = -20.0, 20.0 + for _ in range(100): # mean of a logistic is monotone in the intercept + mid = (lo + hi) / 2 + if (1 / (1 + np.exp(-(z + mid)))).mean() < rate: + lo = mid + else: + hi = mid + return 1 / (1 + np.exp(-(z + (lo + hi) / 2))) + + +def draw_reports(cases: pd.DataFrame, p: np.ndarray, rng: np.random.Generator) -> pd.Series: + """When each case's report arrives (NaT if never), indexed by case id.""" + reported = rng.random(len(cases)) < p + median = cases["typology"].map(DELAY_MEDIAN_DAYS).fillna(DELAY_MEDIAN_DEFAULT_DAYS) + days = rng.lognormal(np.log(median.to_numpy()), 0.8) + rng.uniform(1 / 24, 0.5, len(cases)) + at = cases["started_at"] + pd.to_timedelta(days, unit="D") + return pd.Series(at.where(reported).to_numpy(), index=cases["case_id"].to_numpy()) + + +def observed_labels(frame: pd.DataFrame, report_at: pd.Series, cutoff: pd.Timestamp) -> np.ndarray: + """A fraud-chain row is labelled fraud only if its case was reported before `cutoff`.""" + known = report_at[report_at <= cutoff].index + return (frame["y"].to_numpy() == 1) & frame["case_id"].isin(known).to_numpy() + + +# ---------------------------------------------------------------- recovery + + +def propagate(frame: pd.DataFrame, y_obs: np.ndarray, window: pd.Timedelta = PROPAGATION_WINDOW): + """Rows labelled clean that moved money into or out of a reported mule wallet. + + A wallet that received a reported scam transfer is a known mule from then on. + Other transfers it received, and the money it sent on, within `window` of its + reported transfers inherit a soft fraud label. Recruited mules are ordinary + customers too, so some of what this labels is legitimate. + """ + paid = frame[y_obs & (frame["type"] == "SEND_MONEY").to_numpy()] + span = paid.groupby("receiver_id")["ts"].agg(["min", "max"]) + lo, hi = span["min"] - window, span["max"] + window + hit = np.zeros(len(frame), dtype=bool) + for side in ("receiver_id", "sender_id"): + start = frame[side].map(lo).to_numpy() + end = frame[side].map(hi).to_numpy() + ts = frame["ts"].to_numpy() + hit |= (ts >= start) & (ts <= end) + return hit & ~y_obs + + +def labelling_propensity( + x: np.ndarray, s: np.ndarray, x_val: np.ndarray, s_val: np.ndarray, seed: int +) -> np.ndarray: + """g(x) = P(labelled | x), fitted out-of-fold so no row is scored by a model that saw it.""" + g = np.zeros(len(s)) + folds = StratifiedKFold(PU_FOLDS, shuffle=True, random_state=seed) + for fit_idx, held_idx in folds.split(x, s): + booster = fit_weighted(x[fit_idx], s[fit_idx], None, x_val, s_val, None, seed) + g[held_idx] = booster.predict(x[held_idx]) + return g + + +def elkan_noto(g: np.ndarray, s: np.ndarray, c: float | None = None) -> tuple[np.ndarray, float]: + """Elkan & Noto (2008): how likely each unlabelled row is fraud, and the label rate c. + + c = P(labelled | fraud) is the mean of g over labelled rows unless it is given. An + unlabelled row is fraud with weight (1 - c)/c * g/(1 - g), clipped to [0, 1]; + labelled rows get NaN (they keep their hard label). This assumes labels are + missing at random among frauds, which biased reporting breaks. + """ + if c is None: + c = float(g[s == 1].mean()) + g = np.clip(g, 1e-6, 1 - 1e-6) + w = np.clip((1 - c) / c * g / (1 - g), 0.0, 1.0) + return np.where(s == 1, np.nan, w), c + + +def with_soft_labels(x: np.ndarray, y: np.ndarray, soft: np.ndarray | None): + """Rows whose `soft` is set appear twice: as fraud with that weight, as clean with the rest.""" + if soft is None: + return x, y, None + m = ~np.isnan(soft) & (soft > 1e-4) + keep = ~m + w = soft[m] + return ( + np.vstack([x[keep], x[m], x[m]]), + np.concatenate([y[keep], np.ones(m.sum()), np.zeros(m.sum())]), + np.concatenate([np.ones(keep.sum()), w, 1 - w]), + ) + + +def arrays(part: pd.DataFrame): + """Features, labels and weights of a labelled frame, `soft` expanded by `with_soft_labels`.""" + soft = part["soft"].to_numpy(dtype=float) if "soft" in part else None + if soft is not None and np.isnan(soft).all(): + soft = None + return with_soft_labels(part[list(FEATURES)].to_numpy(), part["y"].to_numpy(float), soft) + + +def fit_weighted(x, y, w, x_val, y_val, w_val, seed: int) -> lgb.Booster: + """`train.fit_booster`, with sample weights and a seed.""" + params = {**LGB_PARAMS, "seed": seed} + dtrain = lgb.Dataset(x, y, weight=w, feature_name=list(FEATURES)) + dval = lgb.Dataset(x_val, y_val, weight=w_val, reference=dtrain) + booster = lgb.train( + params, + dtrain, + num_boost_round=2000, + valid_sets=[dval], + callbacks=[lgb.early_stopping(100, verbose=False)], + ) + return lgb.Booster(model_str=booster.model_to_string(num_iteration=booster.best_iteration)) + + +# ---------------------------------------------------------------- feedback loop + + +def review_window(cfg: dict) -> tuple[int, int, int]: + """Days (fit until, review from, review until) for the model whose alerts get reviewed.""" + train_end = cfg["train_end_day"] + es = (train_end + cfg["val_end_day"]) // 2 - train_end # as long as val_a + review = max(1, round(0.2 * train_end)) + return train_end - review - es, train_end - review, train_end + + +def verdicts( + clean: pd.DataFrame, + report_at: pd.Series, + cfg: dict, + start: pd.Timestamp, + cutoff: pd.Timestamp, + seed: int, +) -> tuple[pd.DataFrame, dict]: + """Analysts' verdicts on the alerts of a model trained on earlier reports. + + The model is fitted on reports known when the review window opens, alerts on the + top REVIEW_BUDGET of the window, and every alert is closed with the true answer + (the same simulated, always-right reviewer as `make review`). + """ + fit_until, review_from, review_until = review_window(cfg) + opens = start + review_from * DAY + y_then = observed_labels(clean, report_at, opens) + day = clean["day"].to_numpy() + fit, stop = day < fit_until, (day >= fit_until) & (day < review_from) + x = clean[list(FEATURES)].to_numpy() + booster = fit_weighted(x[fit], y_then[fit], None, x[stop], y_then[stop], None, seed) + window = clean[(day >= review_from) & (day < review_until)] + score = booster.predict(window[list(FEATURES)].to_numpy()) + alerted = window[score >= np.quantile(score, 1 - REVIEW_BUDGET)].copy() + alerted["y_mule"] = (alerted["y"] * (alerted["type"] == "SEND_MONEY")).astype("int8") + rows = alerted[list(FEEDBACK_COLUMNS)] + truth = alerted["y"].to_numpy() == 1 + unreported = int((truth & ~observed_labels(alerted, report_at, cutoff)).sum()) + return rows, { + "window_days": [review_from, review_until], + "reviewed": int(len(rows)), + "fraud": int(truth.sum()), + "fraud_never_reported_by_training": unreported, + } + + +# ---------------------------------------------------------------- evaluation + + +def operating_fpr(test: pd.DataFrame, models_root: Path) -> dict: + """The served version's false-positive rate on test at its SERVED_TIER threshold.""" + version = registry.current_version(models_root) + if version is None: + raise SystemExit("no served model: run `make train` first") + directory = models_root / version + threshold = json.loads((directory / "manifest.json").read_text())["thresholds"][SERVED_TIER] + scores = pd.read_parquet(directory / "test_scores.parquet", columns=["txn_id", "risk"]) + risk = test[["txn_id", "y"]].merge(scores, on="txn_id", how="inner") + clean = risk[risk["y"] == 0] + return { + "version": version, + "tier": SERVED_TIER, + "threshold": threshold, + "fpr": float((clean["risk"] >= threshold).mean()), + } + + +def evaluate(test: pd.DataFrame, score: np.ndarray, fpr: float) -> dict: + """Ground-truth outcomes when as many clean rows are alerted as the served model alerts.""" + y = test["y"].to_numpy() + threshold = float(np.quantile(score[y == 0], 1 - fpr)) + flagged = score > threshold + out = metrics.alert_outcomes(test, flagged) + return { + "pr_auc": metrics.ranking(y, score)["pr_auc"], + "fpr": round(float(flagged[y == 0].mean()), 5), + "precision": out["precision"], + "loss_txn_recall": out["loss_txn_recall"], + "case_recall": out["case_recall"], + "taka_recall": out["taka_recall"], + "by_typology": { + t: {"loss_txn_recall": v["loss_txn_recall"], "case_recall": v["case_recall"]} + for t, v in out["by_typology"].items() + }, + } + + +def label_summary(part: pd.DataFrame, y_obs: np.ndarray) -> dict: + """How much of the true fraud the training labels contain, by typology.""" + y = part["y"].to_numpy() == 1 + fraud = part[y].assign(labelled=y_obs[y]) + cases = fraud.groupby("case_id").agg( + typology=("typology", "first"), labelled=("labelled", "max") + ) + return { + "fraud_rows": int(y.sum()), + "fraud_rows_labelled": int(y_obs[y].sum()), + "clean_rows_labelled_fraud": int((y_obs & ~y).sum()), + "case_label_rate": metrics._ratio(cases["labelled"].sum(), len(cases)), + "case_label_rate_by_typology": { + str(t): metrics._ratio(g["labelled"].sum(), len(g)) + for t, g in cases.groupby("typology") + }, + } + + +# ---------------------------------------------------------------- experiment + + +def run_seed(clean: pd.DataFrame, cases: pd.DataFrame, cfg: dict, fpr: float, seed: int) -> dict: + start = pd.Timestamp(cfg["start"]) + cutoff = start + cfg["val_end_day"] * DAY # the model is trained when the test period opens + rng = np.random.default_rng(seed) + rate = cfg["report_rate"] + biased_at = draw_reports(cases, report_probability(cases, rate), rng) + uniform_at = draw_reports(cases, report_probability(cases, rate, biased=False), rng) + + fold = clean["fold"].to_numpy() + tr, va, te = fold == "train", fold == "val_a", fold == "test" + x = clean[list(FEATURES)].to_numpy() + test = clean[te] + y_true = clean["y"].to_numpy() + y_rep = observed_labels(clean, biased_at, cutoff) + y_uni = observed_labels(clean, uniform_at, cutoff) + spread = propagate(clean, y_rep) + soft_prop = np.where(spread, PROPAGATION_WEIGHT, np.nan) + + out: dict = {"labels": {}, "regimes": {}, "diagnostics": {}} + out["labels"] = { + name: label_summary(clean[tr | va], y[tr | va]) + for name, y in (("ground_truth", y_true == 1), ("reported_uniform", y_uni), + ("reported", y_rep)) + } # fmt: skip + in_fit = tr | va + out["diagnostics"]["propagation"] = { + "rows_given_soft_label": int(spread[in_fit].sum()), + "of_which_true_fraud": metrics._ratio( + (spread & (y_true == 1))[in_fit].sum(), spread[in_fit].sum() + ), # fmt: skip + } + + def labelled(y: np.ndarray, soft: np.ndarray | None = None) -> pd.DataFrame: + return clean.assign(y=y.astype("int8"), soft=np.nan if soft is None else soft) + + def fit(part: pd.DataFrame) -> np.ndarray: + train, val = part[part["fold"] == "train"], part[part["fold"] == "val_a"] + booster = fit_weighted(*arrays(train), *arrays(val), seed) + return booster.predict(x[te]) + + def with_verdicts(part: pd.DataFrame) -> pd.DataFrame: + # Verdicts replace the labels of the transactions they cover, exactly as + # `train.add_feedback` does when `make retrain` runs. + sub = {"train": part[part["fold"] == "train"], "test": test} + add_feedback(sub, rows) + sub["train"] = sub["train"].assign(fold="train") + return pd.concat([sub["train"], part[part["fold"] == "val_a"]], ignore_index=True) + + out["regimes"]["ground_truth"] = evaluate(test, fit(labelled(y_true)), fpr) + out["regimes"]["reported_uniform"] = evaluate(test, fit(labelled(y_uni)), fpr) + out["regimes"]["reported"] = evaluate(test, fit(labelled(y_rep)), fpr) + + s_tr = y_rep[tr].astype(float) + g = labelling_propensity(x[tr], s_tr, x[va], y_rep[va].astype(float), seed) + soft = np.full(len(clean), np.nan) + soft[tr], c = elkan_noto(g, s_tr) + out["regimes"]["reported_pu"] = evaluate(test, fit(labelled(y_rep, soft)), fpr) + # The same with c taken as the report rate a survey would give, instead of estimated. + soft[tr], _ = elkan_noto(g, s_tr, rate) + out["regimes"]["reported_pu_known_rate"] = evaluate(test, fit(labelled(y_rep, soft)), fpr) + true_c = metrics._ratio(y_rep[tr].sum(), (y_true[tr] == 1).sum()) + out["diagnostics"]["pu"] = {"estimated_label_rate": round(c, 4), "actual_label_rate": true_c} + + propagated = labelled(y_rep, soft_prop) + out["regimes"]["reported_propagated"] = evaluate(test, fit(propagated), fpr) + + rows, out["diagnostics"]["feedback"] = verdicts(clean, biased_at, cfg, start, cutoff, seed) + out["regimes"]["reported_feedback"] = evaluate(test, fit(with_verdicts(labelled(y_rep))), fpr) + out["regimes"]["reported_propagated_feedback"] = evaluate( + test, fit(with_verdicts(propagated)), fpr + ) + return out + + +def summarise(runs: list[dict]) -> dict: + """Mean and standard deviation across seeds of every number in the runs.""" + + def walk(items: list): + first = items[0] + if isinstance(first, list): + return first # a setting, the same for every seed + if isinstance(first, dict): + return {k: walk([i[k] for i in items if k in i]) for k in first} + values = np.array([np.nan if v is None else v for v in items], dtype=float) + sd = float(np.nanstd(values, ddof=1)) if np.isfinite(values).sum() > 1 else 0.0 + return {"mean": round(float(np.nanmean(values)), 4), "sd": round(sd, 4)} + + return walk(runs) + + +def run(data_dir: Path, models_root: Path, seeds: tuple[int, ...] = SEEDS) -> dict: + t0 = time.perf_counter() + frame = load_frame(data_dir) + cfg = json.loads((data_dir / "meta.json").read_text())["config"] + clean = frame[~frame["ambiguous"]].reset_index(drop=True) + cases = case_table(data_dir) + served = operating_fpr(clean[clean["fold"] == "test"], models_root) + runs = {} + for seed in seeds: + runs[seed] = run_seed(clean, cases, cfg, served["fpr"], seed) + print(f"seed {seed}: " + ", ".join( + f"{k} {v['case_recall']:.3f}" for k, v in runs[seed]["regimes"].items() + )) # fmt: skip + return { + "created_at": datetime.now(UTC).isoformat(timespec="seconds"), + "data_dir": str(data_dir), + "data_seed": cfg["seed"], + "seeds": list(seeds), + "operating_point": served, + "assumptions": { + "report_rate": cfg["report_rate"], + "report_logit": REPORT_LOGIT, + "typology_logit": TYPOLOGY_LOGIT, + "delay_median_days": {**DELAY_MEDIAN_DAYS, "other": DELAY_MEDIAN_DEFAULT_DAYS}, + "labels_known_at_day": cfg["val_end_day"], + "propagation_weight": PROPAGATION_WEIGHT, + "propagation_window_days": PROPAGATION_WINDOW / DAY, + "pu_folds": PU_FOLDS, + "review_budget": REVIEW_BUDGET, + }, + "regimes": list(REGIMES), + "summary": summarise(list(runs.values())), + "runs": {str(k): v for k, v in runs.items()}, + "seconds": round(time.perf_counter() - t0, 1), + } + + +def print_summary(report: dict) -> None: + s = report["summary"]["regimes"] + op = report["operating_point"] + print(f"\nat the {op['version']} {op['tier']} tier's false-positive rate, {op['fpr']:.3%}" + f" ({len(report['seeds'])} seeds, mean ± sd):") # fmt: skip + keys = ("pr_auc", "loss_txn_recall", "case_recall", "taka_recall") + rows = {r: {k: f"{s[r][k]['mean']:.3f} ± {s[r][k]['sd']:.3f}" for k in keys} for r in s} + print(pd.DataFrame(rows).T.to_string()) + print("\nscams caught by typology:") + typ = { + r: {t: f"{v['case_recall']['mean']:.3f}" for t, v in s[r]["by_typology"].items()} for r in s + } + print(pd.DataFrame(typ).T.to_string()) + print("\ndiagnostics:", json.dumps(report["summary"]["diagnostics"])) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Train under realistic labels and compare") + parser.add_argument("--small", action="store_true") + parser.add_argument("--data", type=Path, default=None) + parser.add_argument("--seeds", type=int, nargs="+", default=list(SEEDS)) + parser.add_argument( + "--out", type=Path, default=settings.artifacts_dir / "reports" / REPORT_FILE + ) + args = parser.parse_args() + data_dir = args.data or settings.data_dir / ("small" if args.small else "full") + report = run(data_dir, registry.models_dir(), tuple(args.seeds)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2, default=str)) + print_summary(report) + print(f"\nwrote {args.out} in {report['seconds']}s") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/platform/appeals.py b/backend/src/fraudlens/platform/appeals.py new file mode 100644 index 0000000..ab6cd4c --- /dev/null +++ b/backend/src/fraudlens/platform/appeals.py @@ -0,0 +1,130 @@ +"""Customer appeals: "this is a genuine payment", answered by a named person. + +A customer whose payment was warned, put through a step-up or held can say it +is genuine, with how they know the recipient and why. An analyst or supervisor +answers within a deadline: + +- Approving a held payment releases it, through the same path a `legitimate` + verdict uses. Money is never released into a wallet that is confirmed fraud + or frozen, whatever the appeal says. +- Rejecting keeps the hold; the case still decides. +- An appeal against a warning or a step-up moves no money: the customer + already decides those. It records that the warning was wrong, and an approved + one becomes a `legitimate` label (mlops/feedback.py). It never shortens a + cooling-off period: "let it through now, it is genuine" is exactly what a + scammer coaching a victim would have them say. + +What the customer typed is shown masked and is never read to decide anything. +""" + +from __future__ import annotations + +from datetime import timedelta + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from .audit import Ctx, WorkflowError, audit +from .cases import _event +from .models import Appeal, Case, Decision, Transaction +from .scoring import Scorer + +APPEALABLE = ("warn", "step_up", "hold") +# A warning or step-up appeal is feedback: nothing is waiting on it, so a day. +FEEDBACK_SLA_MINUTES = 24 * 60 + + +def sla_minutes(scorer: Scorer, tier: str) -> int: + if tier == "hold": + return scorer.policy.tiers["hold"].review_sla_minutes or FEEDBACK_SLA_MINUTES + return FEEDBACK_SLA_MINUTES + + +def file_appeal( + scorer: Scorer, ctx: Ctx, wallet_id: str, txn_id: int, relation: str, reason: str +) -> Appeal: + with scorer.transaction() as s: + txn = s.scalar( + select(Transaction).where(Transaction.txn_id == txn_id).with_for_update(key_share=True) + ) + # The same answer for "no such transaction" and "not yours". + if txn is None or txn.sender_id != wallet_id: + raise WorkflowError(404, "txn_not_found", "no such transaction for this wallet") + decision = s.get(Decision, txn_id) + if decision is None or decision.tier not in APPEALABLE: + raise WorkflowError( + 409, "not_appealable", "only a warned or held payment can be appealed" + ) + if decision.tier == "hold" and txn.status != "held": + raise WorkflowError(409, "already_decided", f"the payment is already {txn.status}") + now = scorer.now() + appeal = Appeal( + txn_id=txn_id, + wallet_id=wallet_id, + case_id=decision.case_id, + tier=decision.tier, + relation=relation, + reason=reason, + filed_at=now, + sla_due_at=now + timedelta(minutes=sla_minutes(scorer, decision.tier)), + ) + s.add(appeal) + try: + s.flush() + except IntegrityError: + raise WorkflowError( + 409, "appeal_exists", "this payment has already been appealed" + ) from None + if appeal.case_id is not None: + _event( + s, appeal.case_id, None, "appeal_filed", + appeal_id=appeal.id, txn_id=txn_id, relation=relation, tier=decision.tier, + ) # fmt: skip + audit( + s, ctx, "appeal.file", "appeal", appeal.id, + txn_id=txn_id, wallet_id=wallet_id, tier=decision.tier, relation=relation, + ) # fmt: skip + return appeal + + +def decide_appeal(scorer: Scorer, ctx: Ctx, appeal_id: int, approve: bool, note: str) -> dict: + """The person's answer. Approving releases a payment that is still held.""" + with scorer.transaction() as s: + appeal = s.scalar(select(Appeal).where(Appeal.id == appeal_id).with_for_update()) + if appeal is None: + raise WorkflowError(404, "appeal_not_found", f"no appeal {appeal_id}") + if appeal.status != "pending": + raise WorkflowError(409, "already_decided", f"the appeal was {appeal.status}") + if appeal.case_id is not None and ctx.role != "supervisor": + case = s.get(Case, appeal.case_id) + if case is not None and case.status == "escalated": + raise WorkflowError(403, "needs_supervisor", "an escalated case needs a supervisor") + txn = s.scalar( + select(Transaction) + .where(Transaction.txn_id == appeal.txn_id) + .with_for_update(key_share=True) + ) + released = False + if approve and txn.status == "held": + if txn.receiver_id in scorer.engine.flagged: + raise WorkflowError( + 409, "recipient_confirmed_fraud", + "the receiving wallet is confirmed fraud: an appeal cannot release money to it", + ) # fmt: skip + scorer.complete(txn) # a frozen party turns this into `rejected` + released = txn.status == "completed" + now = scorer.now() + appeal.status = "approved" if approve else "rejected" + appeal.decided_by, appeal.decision_note, appeal.decided_at = ctx.user_id, note, now + if appeal.case_id is not None: + _event( + s, appeal.case_id, ctx, f"appeal_{appeal.status}", note, + appeal_id=appeal.id, txn_id=appeal.txn_id, released=released, + ) # fmt: skip + audit( + s, ctx, f"appeal.{'approve' if approve else 'reject'}", "appeal", appeal.id, + txn_id=appeal.txn_id, released=released, txn_status=txn.status, + within_sla=now <= appeal.sla_due_at, + ) # fmt: skip + s.flush() + return {"appeal": appeal, "txn_status": txn.status, "released": released} diff --git a/backend/src/fraudlens/platform/callbacks.py b/backend/src/fraudlens/platform/callbacks.py new file mode 100644 index 0000000..b606997 --- /dev/null +++ b/backend/src/fraudlens/platform/callbacks.py @@ -0,0 +1,178 @@ +"""Decision callbacks: tell the partner what was decided on a transaction it queued. + +Optional, per partner (`callback_url` in the ingest keyring). A queued transaction +puts a job on a Redis sorted set, scored by when it is next due. Any API process +may deliver it: removing the job from the set is the claim, so each attempt +happens once. Until the transaction has been decided the job waits; once it has, +the outcome is POSTed, signed like an ingest request with the partner's newest +key. A failed delivery is retried with exponential backoff and jitter, and after +the last attempt it goes to a dead list for an operator to look at. + +Delivery is at least once (a crash after the POST and before the claim is +dropped repeats it), so the partner should treat `txn_id` as the idempotency key. +""" + +from __future__ import annotations + +import json +import logging +import random +import secrets +import threading +import time +from collections.abc import Callable +from urllib.parse import urlsplit + +import httpx +from redis import Redis +from sqlalchemy.orm import Session, sessionmaker + +from ..config import Settings +from .ingest import signed_headers +from .keys import load_keyring +from .models import Decision, Transaction + +log = logging.getLogger(__name__) + +SCHEDULE = "fraudlens:callbacks" +DEAD = "fraudlens:callbacks:dead" +DEAD_MAXLEN = 10_000 +MAX_ATTEMPTS = 8 +BASE_DELAY = 2.0 # seconds; doubled each attempt +MAX_DELAY = 600.0 +POLL_DELAY = 2.0 # while the transaction is still in the stream +DECISION_WAIT = 600.0 # after this, report it as not processed (dead-lettered in the stream) +TIMEOUT = 5.0 + +Post = Callable[[str, dict[str, str], bytes], int] + + +def schedule(redis: Redis, partner: str, txn_id: int, end_to_end_id: str, now: float) -> None: + job = {"partner": partner, "txn_id": txn_id, "e2e": end_to_end_id, "queued": now, "try": 0} + redis.zadd(SCHEDULE, {json.dumps(job, sort_keys=True): now + POLL_DELAY}) + + +def backoff(attempt: int) -> float: + """Seconds before attempt `attempt + 1`: 2, 4, 8 ... capped, with ±20% jitter.""" + return min(MAX_DELAY, BASE_DELAY * 2**attempt) * random.uniform(0.8, 1.2) + + +def _http_post(url: str, headers: dict[str, str], body: bytes) -> int: + with httpx.Client(timeout=TIMEOUT, follow_redirects=False) as client: + return client.post(url, content=body, headers=headers).status_code + + +class Dispatcher(threading.Thread): + def __init__( + self, + settings: Settings, + sessions: sessionmaker[Session], + redis: Redis, + post: Post | None = None, + interval: float = 1.0, + ) -> None: + super().__init__(name="fraudlens-callbacks", daemon=True) + self.settings, self.sessions, self.redis = settings, sessions, redis + self.post = post or _http_post + self.interval = interval + self.delivered = self.failed = self.dead = 0 + self._stopping = threading.Event() + + def outcome(self, txn_id: int) -> dict | None: + """What the partner is told, or None while the transaction is not decided.""" + with self.sessions() as s: + txn = s.get(Transaction, txn_id) + if txn is None: + return None + decision = s.get(Decision, txn_id) + summary = None + if decision is not None: # what to do, not why: as /v1/score tells the service + summary = { + "tier": decision.tier, + "action": decision.action, + "requires_review": decision.requires_review, + "risk_score": decision.risk_score, + "risk_band": decision.risk_band, + "mode": decision.mode, + "model_version": decision.model_version, + "policy_version": decision.policy_version, + "decided_at": decision.decided_at.isoformat(), + } + return { + "status": txn.status, + "status_reason": txn.status_reason, + "scored": decision is not None, + "decision": summary, + } + + def _later(self, job: dict, at: float) -> None: + self.redis.zadd(SCHEDULE, {json.dumps(job, sort_keys=True): at}) + + def _deliver(self, job: dict, now: float) -> None: + outcome = self.outcome(job["txn_id"]) + if outcome is None: + if now - job["queued"] < DECISION_WAIT: + self._later(job, now + POLL_DELAY) + return + outcome = {"status": "not_processed", "scored": False, "decision": None} + ring = load_keyring(self.settings.ingest_keyring) + url, key = ring.callback_url(job["partner"]), ring.signing(job["partner"]) + if not url or key is None: # the callback was switched off, or every key retired + log.info("callback for %s dropped: no url or key", job["partner"]) + return + payload = {"txn_id": job["txn_id"], "end_to_end_id": job["e2e"], **outcome} + body = json.dumps(payload, separators=(",", ":")).encode() + parts = urlsplit(url) + path = parts.path or "/" + path += f"?{parts.query}" if parts.query else "" + headers = signed_headers(key, "POST", path, body, secrets.token_hex(16)) + try: + status = self.post(url, headers, body) + except httpx.HTTPError as exc: + status, error = 0, type(exc).__name__ + else: + error = f"HTTP {status}" + if 200 <= status < 300: + self.delivered += 1 + return + self.failed += 1 + job = job | {"try": job["try"] + 1, "error": error} + if job["try"] >= MAX_ATTEMPTS: + self.dead += 1 + log.warning( + "callback for txn %s to %s gave up: %s", job["txn_id"], job["partner"], error + ) + pipe = self.redis.pipeline(transaction=False) + pipe.lpush(DEAD, json.dumps(job | {"payload": payload}, sort_keys=True)) + pipe.ltrim(DEAD, 0, DEAD_MAXLEN - 1) + pipe.execute() + return + self._later(job, now + backoff(job["try"])) + + def run_once(self, now: float | None = None) -> int: + """Handle the jobs that are due. Returns how many were taken.""" + now = time.time() if now is None else now + due = self.redis.zrangebyscore(SCHEDULE, "-inf", now, start=0, num=50) + taken = 0 + for member in due: + if not self.redis.zrem(SCHEDULE, member): + continue # another process claimed it + taken += 1 + job = json.loads(member) + try: + self._deliver(job, now) + except Exception: + log.exception("callback for txn %s failed; retrying", job.get("txn_id")) + self._later(job, now + backoff(job.get("try", 0))) + return taken + + def run(self) -> None: + while not self._stopping.is_set(): + try: + self.run_once() + except Exception: + log.exception("callback dispatcher error") + self._stopping.wait(self.interval) + + def stop(self) -> None: + self._stopping.set() diff --git a/backend/src/fraudlens/platform/ingest.py b/backend/src/fraudlens/platform/ingest.py new file mode 100644 index 0000000..b2362f7 --- /dev/null +++ b/backend/src/fraudlens/platform/ingest.py @@ -0,0 +1,342 @@ +"""The signed ingest webhook: a core-banking system pushes transactions to FraudLens. + +The protocol is in docs/INGEST.md. In short, every request carries + + X-FraudLens-Key-Id: upay-20261007-77d1 which partner key signed it + X-FraudLens-Timestamp: 1791364502 Unix seconds, within ±5 minutes of ours + X-FraudLens-Nonce: 6f1c0e5d9a2b4c7e8f90 16-64 characters, used once + X-FraudLens-Signature: v1= over timestamp, nonce, method, path, body + +and the body is a pacs.008-shaped JSON message (ISO 20022 FI-to-FI customer +credit transfer). Each `CdtTrfTxInf` maps onto the same `TxnIn` that /v1/score +takes, so a transaction is validated by exactly one set of rules whichever way +it arrives. The same signature scheme, with the partner's newest key, signs the +decision callbacks going the other way. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import re +import time +from collections.abc import Mapping +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, IPvAnyAddress, ValidationError +from redis import Redis + +from .audit import WorkflowError +from .events import TxnIn +from .keys import Key, Keyring + +KEY_HEADER = "X-FraudLens-Key-Id" +TIMESTAMP_HEADER = "X-FraudLens-Timestamp" +NONCE_HEADER = "X-FraudLens-Nonce" +SIGNATURE_HEADER = "X-FraudLens-Signature" +IDEMPOTENCY_HEADER = "Idempotency-Key" +SCHEME = "v1" + +_NONCE = re.compile(r"^[A-Za-z0-9_-]{16,64}$") +_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9_.:-]{1,128}$") +_SIGNATURE = re.compile(r"^[0-9a-f]{64}$") +IDEMPOTENCY_TTL = 24 * 3600 +SEEN_TXN_TTL = 7 * 24 * 3600 +MAX_BATCH = 500 +_PREFIX = "fraudlens:ingest" + +# ---------------------------------------------------------------- signatures + + +def canonical(timestamp: str, nonce: str, method: str, path: str, body: bytes) -> bytes: + """What is signed. The method and path (with its query) are in it, so a request + signed for one endpoint, or for a queued answer, cannot be replayed at another.""" + return f"{timestamp}\n{nonce}\n{method.upper()}\n{path}\n".encode() + body + + +def sign(secret: str, timestamp: str, nonce: str, method: str, path: str, body: bytes) -> str: + mac = hmac.new(secret.encode(), canonical(timestamp, nonce, method, path, body), "sha256") + return f"{SCHEME}={mac.hexdigest()}" + + +def signed_headers(key: Key, method: str, path: str, body: bytes, nonce: str) -> dict[str, str]: + """Headers for a request this side sends (the decision callback).""" + timestamp = str(int(time.time())) + return { + KEY_HEADER: key.kid, + TIMESTAMP_HEADER: timestamp, + NONCE_HEADER: nonce, + SIGNATURE_HEADER: sign(key.secret, timestamp, nonce, method, path, body), + "Content-Type": "application/json", + } + + +def _denied(code: str, message: str) -> WorkflowError: + return WorkflowError(401, code, message) + + +def verify( + ring: Keyring, + headers: Mapping[str, str], + method: str, + path: str, + body: bytes, + window: int, + now: float | None = None, +) -> Key: + """The partner key that signed this request, or a 401 saying what is wrong. + The answers name the check that failed and never what was expected.""" + kid, timestamp = headers.get(KEY_HEADER), headers.get(TIMESTAMP_HEADER) + nonce, signature = headers.get(NONCE_HEADER), headers.get(SIGNATURE_HEADER) + if not (kid and timestamp and nonce and signature): + raise _denied( + "unsigned", f"{KEY_HEADER}, {TIMESTAMP_HEADER}, {NONCE_HEADER} and " + f"{SIGNATURE_HEADER} are required", + ) # fmt: skip + key = ring.verifying(kid) + if key is None or key.partner is None: + raise _denied("unknown_key", "the key id is unknown or retired") + if not timestamp.isdigit() or len(timestamp) > 12: + raise _denied("stale_timestamp", "the timestamp must be Unix seconds") + now = time.time() if now is None else now + if abs(now - int(timestamp)) > window: + raise _denied("stale_timestamp", f"the timestamp must be within {window} seconds of now") + if not _NONCE.fullmatch(nonce): + raise _denied("bad_nonce", "the nonce must be 16 to 64 letters, digits, - or _") + expected = sign(key.secret, timestamp, nonce, method, path, body) + offered = [s.strip() for s in signature.split(",")] # several during a scheme change + if not any( + s.startswith(f"{SCHEME}=") + and _SIGNATURE.fullmatch(s[len(SCHEME) + 1 :]) + and hmac.compare_digest(s, expected) + for s in offered + ): + raise _denied("bad_signature", "the signature does not match the request") + return key + + +def claim_nonce(redis: Redis, key: Key, nonce: str, window: int) -> None: + """A signed request is accepted once. The nonce is remembered for twice the + timestamp window, after which the timestamp check refuses the request anyway.""" + if not redis.set(f"{_PREFIX}:nonce:{key.kid}:{nonce}", "1", nx=True, ex=2 * window): + raise WorkflowError(409, "replayed", "this signed request has already been received") + + +# ---------------------------------------------------------------- idempotency + + +class Idempotency: + """`Idempotency-Key`: a retry with the same key and body gets the first answer back. + + The key is per partner. The same key with a different body is refused, so a key + cannot be reused by mistake for another payment.""" + + def __init__(self, redis: Redis, partner: str, value: str | None, body: bytes) -> None: + if value is not None and not _IDEMPOTENCY_KEY.fullmatch(value): + raise WorkflowError( + 400, "bad_idempotency_key", "Idempotency-Key must be 1-128 of A-Z a-z 0-9 _ . : -" + ) + self.redis = redis + self.key = f"{_PREFIX}:idem:{partner}:{value}" if value else None + self.digest = hashlib.sha256(body).hexdigest() + + def begin(self) -> tuple[int, dict] | None: + """The stored answer to replay, or None when this request is the first.""" + if self.key is None: + return None + marker = json.dumps({"digest": self.digest}) + if self.redis.set(self.key, marker, nx=True, ex=IDEMPOTENCY_TTL): + return None + stored = json.loads(self.redis.get(self.key) or marker) + if stored["digest"] != self.digest: + raise WorkflowError( + 422, "idempotency_key_reused", "this Idempotency-Key was used for another body" + ) + if "status" not in stored: + raise WorkflowError( + 409, "in_progress", "a request with this Idempotency-Key is being processed", + retry_after_seconds=1, + ) # fmt: skip + return stored["status"], stored["body"] + + def finish(self, status: int, body: dict) -> None: + if self.key is not None: + record = {"digest": self.digest, "status": status, "body": body} + self.redis.set(self.key, json.dumps(record, default=str), ex=IDEMPOTENCY_TTL) + + def abandon(self) -> None: + """The request failed before an answer worth keeping: a retry may run it again.""" + if self.key is not None: + self.redis.delete(self.key) + + +def first_sightings(redis: Redis, txn_ids: list[int]) -> list[bool]: + """For each transaction id, whether this is the first time ingest has seen it.""" + pipe = redis.pipeline(transaction=False) + for txn_id in txn_ids: + pipe.set(f"{_PREFIX}:txn:{txn_id}", "1", nx=True, ex=SEEN_TXN_TTL) + return [bool(new) for new in pipe.execute()] + + +def forget_sightings(redis: Redis, txn_ids: list[int]) -> None: + if txn_ids: + redis.delete(*(f"{_PREFIX}:txn:{txn_id}" for txn_id in txn_ids)) + + +# ---------------------------------------------------------------- pacs.008 + + +Max35Text = Field(pattern=r"^[A-Za-z0-9/:._-]{1,35}$") + + +class _Iso(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class GroupHeader(_Iso): + MsgId: str = Max35Text + CreDtTm: datetime + NbOfTxs: int | None = Field(default=None, ge=1, le=MAX_BATCH) + + +class PaymentId(_Iso): + EndToEndId: str = Max35Text + TxId: str = Field(pattern=r"^[0-9]{1,18}$") # the platform's numeric transaction id + + +class Proprietary(_Iso): + Prtry: str = Field(max_length=35) + + +class CategoryPurpose(_Iso): + CtgyPurp: Proprietary # SEND_MONEY, CASH_OUT, ... + + +class Amount(_Iso): + Ccy: Literal["BDT"] + Amt: float = Field(gt=0, le=10_000_000, allow_inf_nan=False) + + +class SchemeName(_Iso): + Prtry: Literal["wallet", "agent", "bank", "merchant", "telco", "biller"] + + +class OtherId(_Iso): + Id: str = Field(pattern=r"^[A-Za-z0-9_-]{1,32}$") + SchmeNm: SchemeName + + +class AccountId(_Iso): + Othr: OtherId + + +class Account(_Iso): + Id: AccountId + + +class Envelope(_Iso): + """Context a credit transfer does not carry, but the channel knows.""" + + DbtrBalBefore: float | None = Field(default=None, ge=0, le=1e10, allow_inf_nan=False) + DvcId: str = Field(default="", max_length=64, pattern=r"^[A-Za-z0-9_-]*$") + Chanl: Literal["app", "ussd", "agent", "bank"] + Dstrct: str = Field(min_length=1, max_length=40, pattern=r"^[A-Za-z' .-]+$") + IPAddr: IPvAnyAddress | None = None + + +class SupplementaryData(_Iso): + Envlp: Envelope + + +class CreditTransfer(_Iso): + PmtId: PaymentId + PmtTpInf: CategoryPurpose + IntrBkSttlmAmt: Amount + AccptncDtTm: datetime # when the customer confirmed: the transaction's time + DbtrAcct: Account + CdtrAcct: Account + SplmtryData: SupplementaryData + + def txn(self) -> TxnIn: + """The platform's own event. Raises ValidationError with the platform's rules.""" + debtor, creditor = self.DbtrAcct.Id.Othr, self.CdtrAcct.Id.Othr + extra = self.SplmtryData.Envlp + return TxnIn.model_validate( + { + "txn_id": int(self.PmtId.TxId), + "ts": self.AccptncDtTm, + "type": self.PmtTpInf.CtgyPurp.Prtry, + "sender_id": debtor.Id, + "sender_type": debtor.SchmeNm.Prtry, + "receiver_id": creditor.Id, + "receiver_type": creditor.SchmeNm.Prtry, + "amount": self.IntrBkSttlmAmt.Amt, + "sender_balance_before": extra.DbtrBalBefore, + "device_id": extra.DvcId, + "channel": extra.Chanl, + "district": extra.Dstrct, + "ip": extra.IPAddr, + "source": "live", + } + ) + + +# TxnIn field -> where it sits in a CdtTrfTxInf. docs/INGEST.md has the same table. +ISO_PATH = { + "txn_id": ("PmtId", "TxId"), + "ts": ("AccptncDtTm",), + "type": ("PmtTpInf", "CtgyPurp", "Prtry"), + "sender_id": ("DbtrAcct", "Id", "Othr", "Id"), + "sender_type": ("DbtrAcct", "Id", "Othr", "SchmeNm", "Prtry"), + "receiver_id": ("CdtrAcct", "Id", "Othr", "Id"), + "receiver_type": ("CdtrAcct", "Id", "Othr", "SchmeNm", "Prtry"), + "amount": ("IntrBkSttlmAmt", "Amt"), + "sender_balance_before": ("SplmtryData", "Envlp", "DbtrBalBefore"), + "device_id": ("SplmtryData", "Envlp", "DvcId"), + "channel": ("SplmtryData", "Envlp", "Chanl"), + "district": ("SplmtryData", "Envlp", "Dstrct"), + "ip": ("SplmtryData", "Envlp", "IPAddr"), +} + + +class Pacs008(_Iso): + """FIToFICstmrCdtTrf, in JSON, with the elements FraudLens uses.""" + + GrpHdr: GroupHeader + CdtTrfTxInf: list[CreditTransfer] = Field(min_length=1, max_length=MAX_BATCH) + + +def parse(body: bytes, single: bool) -> tuple[Pacs008, list[TxnIn]]: + """The message and its transactions, or the `ValidationError`s with their place + in the message. Every transaction is checked before any is accepted.""" + message = Pacs008.model_validate_json(body) + count = len(message.CdtTrfTxInf) + problems: list[dict] = [] + if single and count != 1: + problems.append( + {"loc": ("CdtTrfTxInf",), "msg": "exactly one transaction; use /batch for more"} + ) + if message.GrpHdr.NbOfTxs is not None and message.GrpHdr.NbOfTxs != count: + problems.append({"loc": ("GrpHdr", "NbOfTxs"), "msg": "does not match CdtTrfTxInf"}) + txns = [] + for i, transfer in enumerate(message.CdtTrfTxInf): + if int(transfer.PmtId.TxId) >= 2**62: + problems.append({"loc": ("CdtTrfTxInf", i, "PmtId", "TxId"), "msg": "too large"}) + continue + try: + txns.append(transfer.txn()) + except ValidationError as exc: + for e in exc.errors(): # reported where the partner put it, not by our name + where = ISO_PATH.get(str(e["loc"][0]), ()) if e["loc"] else () + problems.append({"loc": ("CdtTrfTxInf", i, *where), "msg": e["msg"]}) + if problems: + raise IngestInvalid(problems) + return message, txns + + +class IngestInvalid(Exception): + def __init__(self, errors: list[dict]) -> None: + super().__init__(f"{len(errors)} problems") + self.errors = errors diff --git a/backend/src/fraudlens/platform/keys.py b/backend/src/fraudlens/platform/keys.py new file mode 100644 index 0000000..7ee492a --- /dev/null +++ b/backend/src/fraudlens/platform/keys.py @@ -0,0 +1,269 @@ +"""Keyrings: signing keys with ids, so a key can be replaced without a flag day. + +A keyring is a JSON file, kept wherever the deployment keeps secrets (a mounted +Docker or Kubernetes secret, a file only the service user can read): + + { + "keys": { + "jwt-20261007-3fa2": {"secret": "...", "created": "2026-10-07T09:00:00Z"}, + "jwt-20260901-91c0": {"secret": "...", "created": "...", "retire_at": "..."}, + "upay-20261007-77d1": {"secret": "...", "created": "...", "partner": "upay"} + }, + "partners": {"upay": {"callback_url": "https://core.upay.example/fraudlens"}} + } + +The newest unretired key signs; every unretired key verifies. A key past its +`retire_at` verifies nothing. The service re-reads the file when it changes, so a +rotation needs no restart. The JWT keyring holds keys without a partner; the +ingest keyring holds each partner's keys and, optionally, where to call it back. + + python -m fraudlens.platform.keys rotate-jwt secrets/jwt_keys.json + python -m fraudlens.platform.keys rotate-partner secrets/ingest_keys.json --partner upay + python -m fraudlens.platform.keys retire secrets/ingest_keys.json upay-20260901-91c0 + python -m fraudlens.platform.keys list secrets/ingest_keys.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import secrets +import tempfile +import threading +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from pathlib import Path + +KID = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") +PARTNER = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$") +JWT_PREFIX = "jwt" + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _iso(moment: datetime) -> str: + return moment.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _parse(value: str | None) -> datetime | None: + if value is None: + return None + moment = datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=UTC) + + +@dataclass(frozen=True) +class Key: + kid: str + secret: str = field(repr=False) + created: datetime + partner: str | None = None + retire_at: datetime | None = None + + def active(self, now: datetime | None = None) -> bool: + return self.retire_at is None or (now or _now()) < self.retire_at + + +@dataclass +class Keyring: + keys: dict[str, Key] = field(default_factory=dict) + partners: dict[str, dict] = field(default_factory=dict) + + # ------------------------------------------------------------- reading + + def verifying(self, kid: str, now: datetime | None = None) -> Key | None: + key = self.keys.get(kid) + return key if key is not None and key.active(now) else None + + def signing(self, partner: str | None = None, now: datetime | None = None) -> Key | None: + """The newest active key of this partner (None: the keys without a partner).""" + own = [k for k in self.keys.values() if k.partner == partner and k.active(now)] + return max(own, key=lambda k: (k.created, k.kid)) if own else None + + def callback_url(self, partner: str) -> str | None: + return (self.partners.get(partner) or {}).get("callback_url") + + # ------------------------------------------------------------- changing + + def add(self, prefix: str, partner: str | None = None, now: datetime | None = None) -> Key: + now = now or _now() + while True: + kid = f"{prefix}-{now:%Y%m%d}-{secrets.token_hex(2)}" + if kid not in self.keys: + break + key = Key(kid, secrets.token_hex(32), now, partner) + self.keys[kid] = key + return key + + def retire(self, kid: str, at: datetime) -> None: + key = self.keys[kid] + if key.retire_at is None or at < key.retire_at: + self.keys[kid] = Key(key.kid, key.secret, key.created, key.partner, at) + + def prune(self, now: datetime | None = None) -> list[str]: + """Drop keys already retired: nothing can be verified with them any more.""" + gone = [kid for kid, key in self.keys.items() if not key.active(now)] + for kid in gone: + del self.keys[kid] + return gone + + # ------------------------------------------------------------- storage + + @classmethod + def from_json(cls, text: str) -> Keyring: + raw = json.loads(text) + ring = cls(partners=dict(raw.get("partners") or {})) + for kid, entry in (raw.get("keys") or {}).items(): + if not KID.fullmatch(kid): + raise ValueError(f"key id {kid!r} is not valid") + partner = entry.get("partner") + if partner is not None and not PARTNER.fullmatch(partner): + raise ValueError(f"partner name {partner!r} is not valid") + secret = entry["secret"] + if not isinstance(secret, str) or not secret: + raise ValueError(f"key {kid} has no secret") + created = _parse(entry.get("created")) or datetime(1970, 1, 1, tzinfo=UTC) + ring.keys[kid] = Key(kid, secret, created, partner, _parse(entry.get("retire_at"))) + return ring + + def to_json(self) -> str: + keys = {} + for kid, k in sorted(self.keys.items()): + entry = {"secret": k.secret, "created": _iso(k.created)} + if k.partner: + entry["partner"] = k.partner + if k.retire_at: + entry["retire_at"] = _iso(k.retire_at) + keys[kid] = entry + return json.dumps({"keys": keys, "partners": self.partners}, indent=2) + "\n" + + def save(self, path: Path) -> None: + """Write atomically, readable by the owner only.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.") + try: + with os.fdopen(fd, "w") as out: + out.write(self.to_json()) + os.chmod(tmp, 0o600) + os.replace(tmp, path) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + + +_cache: dict[Path, tuple[tuple[int, int], Keyring]] = {} +_cache_lock = threading.Lock() + + +def load_keyring(path: Path) -> Keyring: + """The keyring at `path`, re-read only when the file has changed.""" + path = Path(path) + stat = path.stat() + stamp = (stat.st_mtime_ns, stat.st_size) + with _cache_lock: + cached = _cache.get(path) + if cached and cached[0] == stamp: + return cached[1] + ring = Keyring.from_json(path.read_text()) + with _cache_lock: + _cache[path] = (stamp, ring) + return ring + + +def _open(path: Path) -> Keyring: + return Keyring.from_json(path.read_text()) if path.exists() else Keyring() + + +def rotate_jwt(path: Path, ttl_minutes: int, now: datetime | None = None) -> Key: + """A new signing key. The old ones keep verifying until the tokens they signed expire.""" + now = now or _now() + ring = _open(path) + ring.prune(now) + for key in list(ring.keys.values()): + if key.partner is None: + ring.retire(key.kid, now + timedelta(minutes=ttl_minutes, seconds=60)) + new = ring.add(JWT_PREFIX, now=now) + ring.save(path) + return new + + +def rotate_partner( + path: Path, + partner: str, + grace: timedelta, + callback_url: str | None = None, + now: datetime | None = None, +) -> Key: + """A new key for a partner. Its old keys stay valid for `grace`, time to switch over.""" + if not PARTNER.fullmatch(partner): + raise ValueError(f"partner name {partner!r} is not valid") + now = now or _now() + ring = _open(path) + ring.prune(now) + for key in list(ring.keys.values()): + if key.partner == partner: + ring.retire(key.kid, now + grace) + new = ring.add(partner, partner, now) + entry = ring.partners.setdefault(partner, {}) + if callback_url is not None: + entry["callback_url"] = callback_url or None + ring.save(path) + return new + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + jwt_cmd = sub.add_parser( + "rotate-jwt", help="new token-signing key; old ones verify until expiry" + ) + jwt_cmd.add_argument("path", type=Path) + jwt_cmd.add_argument("--ttl-minutes", type=int, default=None, help="default: from settings") + partner_cmd = sub.add_parser("rotate-partner", help="new HMAC key for an ingest partner") + partner_cmd.add_argument("path", type=Path) + partner_cmd.add_argument("--partner", required=True) + partner_cmd.add_argument("--grace-hours", type=float, default=24.0) + partner_cmd.add_argument("--callback-url", default=None, help="'' removes it") + retire_cmd = sub.add_parser("retire", help="stop accepting one key now (a leaked key)") + retire_cmd.add_argument("path", type=Path) + retire_cmd.add_argument("kid") + list_cmd = sub.add_parser("list", help="key ids and their dates, never the secrets") + list_cmd.add_argument("path", type=Path) + args = parser.parse_args(argv) + + if args.command == "rotate-jwt": + if args.ttl_minutes is None: + from ..config import Settings + + args.ttl_minutes = Settings().jwt_ttl_minutes + key = rotate_jwt(args.path, args.ttl_minutes) + print(f"signing with {key.kid}; earlier keys verify for {args.ttl_minutes} more minutes") + elif args.command == "rotate-partner": + grace = timedelta(hours=args.grace_hours) + key = rotate_partner(args.path, args.partner, grace, args.callback_url) + # The one time the secret is shown: hand it to the partner over a secure channel. + print(f"key id: {key.kid}\nsecret: {key.secret}") + print(f"the partner's earlier keys stay valid for {args.grace_hours:g} hours") + elif args.command == "retire": + ring = _open(args.path) + if args.kid not in ring.keys: + parser.error(f"no key {args.kid} in {args.path}") + ring.retire(args.kid, _now()) + ring.save(args.path) + print(f"{args.kid} is no longer accepted") + else: + ring = _open(args.path) + now = _now() + for kid, k in sorted(ring.keys.items(), key=lambda item: item[1].created): + state = "active" if k.active(now) else "retired" + until = f" until {_iso(k.retire_at)}" if k.retire_at and k.active(now) else "" + owner = k.partner or "tokens" + print(f"{kid:28} {owner:12} created {_iso(k.created)} {state}{until}") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/platform/load.py b/backend/src/fraudlens/platform/load.py index 1471a47..d363a2f 100644 --- a/backend/src/fraudlens/platform/load.py +++ b/backend/src/fraudlens/platform/load.py @@ -28,12 +28,13 @@ from .audit import Ctx, audit from .db import make_engine, migrate from .models import Transaction -from .stream import DEAD_SUFFIX +from .stream import RESET_SUFFIXES SYSTEM = Ctx(None, "system", None) # Everything the platform writes at run time. Never users, never the audit log. RUNTIME_TABLES = ( - "shadow_scores", "decisions", "case_events", "freeze_requests", "customer_reports", + "appeals", "pending_decisions", "shadow_scores", "decisions", "case_events", + "freeze_requests", "customer_reports", "wallet_flags", "cases", "transactions", "past_cases", "wallets", "agents", ) # fmt: skip TXN_COLUMNS = ( @@ -118,7 +119,7 @@ def load(engine: Engine, redis: Redis | None, settings: Settings, reset: bool = c.execute(text("ANALYZE")) if redis is not None and reset: # Events queued for the previous contents make no sense against the new ones. - redis.delete(settings.events_stream, settings.events_stream + DEAD_SUFFIX) + redis.delete(*(settings.events_stream + suffix for suffix in RESET_SUFFIXES)) return {**counts, "history_ends": str(cutoff), "seconds": round(time.perf_counter() - t0, 1)} diff --git a/backend/src/fraudlens/platform/loadtest.py b/backend/src/fraudlens/platform/loadtest.py new file mode 100644 index 0000000..c33446b --- /dev/null +++ b/backend/src/fraudlens/platform/loadtest.py @@ -0,0 +1,251 @@ +"""Load test: 1, 2, 4... stream workers on the same slice of the test period. + + uv run python -m fraudlens.platform.loadtest --workers 1 2 4 + +For each worker count, on a database reset to the historical period: + +1. start the workers (`fraudlens.platform.worker`, one process each) and wait + until each has rebuilt its state; +2. saturation: write the first `--saturation` events at once and time how long the + workers take to decide them all (throughput); +3. open loop: write the next `--rate-events` at a fixed `--rate` per second and + measure end-to-end latency, from the entry being added to its decision being + committed, as the workers record it; +4. stop the workers and compare every decision with the offline evaluation + (`verify`): tier mismatches and differing feature rows must both be 0. + +It writes the JSON to `--out` and prints a markdown table. Destructive: it resets +the database named by FRAUDLENS_DATABASE_URL. Point it at a scratch one. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import os +import platform +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import numpy as np +from redis import Redis + +from ..config import BACKEND_DIR, Settings +from . import verify +from .db import make_engine, migrate +from .load import load +from .replay import test_events +from .stream import GROUP + +BASE_PORT = 18100 + + +def _wait_ready(ports: list[int], procs: list[subprocess.Popen], timeout: float = 900) -> None: + deadline = time.monotonic() + timeout + pending = set(ports) + while pending: + if time.monotonic() > deadline: + raise SystemExit(f"workers on {sorted(pending)} not ready after {timeout:.0f}s") + for proc in procs: + if proc.poll() is not None: + raise SystemExit(f"a worker exited with code {proc.returncode}") + for port in list(pending): + try: + if httpx.get(f"http://127.0.0.1:{port}/ready", timeout=1).status_code == 200: + pending.discard(port) + except httpx.HTTPError: + pass + time.sleep(0.5) + + +def _drained(redis: Redis, stream: str, timeout: float = 1800) -> float: + """Wait until every entry is decided and deleted; returns when that was seen.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if redis.xlen(stream) == 0: + try: + pending = redis.xpending(stream, GROUP)["pending"] + except Exception: + pending = 0 + if pending == 0: + return time.time() + time.sleep(0.02) + raise SystemExit("the stream did not drain") + + +def _write(redis: Redis, stream: str, events: list[dict], rate: float = 0.0) -> None: + """All at once, or paced in 10 ms ticks (open loop: never waits for the workers).""" + if not rate: + pipe = redis.pipeline(transaction=False) + for i, event in enumerate(events, 1): + pipe.xadd(stream, {"data": json.dumps(event)}) + if i % 2000 == 0: + pipe.execute() + pipe.execute() + return + t0, sent = time.perf_counter(), 0 + while sent < len(events): + due = min(len(events), int((time.perf_counter() - t0) * rate) + 1) + if due > sent: + pipe = redis.pipeline(transaction=False) + for event in events[sent:due]: + pipe.xadd(stream, {"data": json.dumps(event)}) + pipe.execute() + sent = due + time.sleep(0.01) + + +def _latency(samples: list[float]) -> dict: + if not samples: + return {} + p50, p95, p99 = np.percentile(np.array(samples) * 1000, [50, 95, 99]) + return { + "p50": round(float(p50), 1), + "p95": round(float(p95), 1), + "p99": round(float(p99), 1), + "max": round(max(samples) * 1000, 1), + "samples": len(samples), + } + + +def run(settings: Settings, n: int, events: list[dict], saturation: int, rate: float) -> dict: + stream = settings.events_stream + engine = make_engine(settings.database_url) + redis = Redis.from_url(settings.redis_url, decode_responses=True) + print(f"\n== {n} worker(s): resetting the database ==", flush=True) + load(engine, redis, settings, reset=True) + + ports = [BASE_PORT + i for i in range(n)] + env = os.environ | {"FRAUDLENS_RUN_WORKER": "false"} + procs = [ + subprocess.Popen( + [ + sys.executable, + "-m", + "fraudlens.platform.worker", + "--port", + str(port), + "--name", + f"loadtest-{n}-{i}", + ], + cwd=BACKEND_DIR, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + for i, port in enumerate(ports) + ] + try: + t_start = time.monotonic() + _wait_ready(ports, procs) + startup = time.monotonic() - t_start + print(f" ready in {startup:.0f}s; saturation run of {saturation:,} events", flush=True) + + t0 = time.time() + _write(redis, stream, events[:saturation]) + t1 = _drained(redis, stream) + throughput = saturation / (t1 - t0) + print(f" {throughput:,.0f} events/s; open loop at {rate:g}/s", flush=True) + + t2 = time.time() + rest = events[saturation:] + _write(redis, stream, rest, rate=rate) + _drained(redis, stream) + + samples, per_worker = [], [] + for port in ports: + m = httpx.get(f"http://127.0.0.1:{port}/metrics?raw=1", timeout=30).json() + samples.extend(lat for at, lat in m.pop("samples") if at >= t2) + per_worker.append(m) + finally: + for proc in procs: + proc.terminate() + for proc in procs: + try: + proc.wait(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + report = verify.verify(engine, settings) + engine.dispose() + result = { + "workers": n, + "startup_seconds": round(startup, 1), + "saturation": { + "events": saturation, + "seconds": round(t1 - t0, 2), + "events_per_second": round(throughput, 1), + }, + "open_loop": { + "events": len(rest), + "rate_per_second": rate, + "latency_ms": _latency(samples), + }, + "per_worker": per_worker, + "parity": { + k: report.get(k) + for k in ("ok", "compared", "tier_mismatches", "feature_rows_differing") + }, + } + print(json.dumps(result | {"per_worker": "..."}, indent=1), flush=True) + return result + + +def table(results: list[dict]) -> str: + base = results[0]["saturation"]["events_per_second"] + lines = [ + "| Workers | Throughput (events/s) | Speed-up | p50 (ms) | p95 (ms) | p99 (ms) " + "| Tier mismatches | Differing feature rows |", + "|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for r in results: + lat, par = r["open_loop"]["latency_ms"], r["parity"] + eps = r["saturation"]["events_per_second"] + lines.append( + f"| {r['workers']} | {eps:,.0f} | {eps / base:.2f}x | {lat.get('p50')} | " + f"{lat.get('p95')} | {lat.get('p99')} | {par['tier_mismatches']} | " + f"{par['feature_rows_differing']} |" + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--workers", type=int, nargs="+", default=[1, 2, 4]) + parser.add_argument("--saturation", type=int, default=12_000) + parser.add_argument("--rate", type=float, default=150.0, help="open-loop events per second") + parser.add_argument("--rate-events", type=int, default=6_000) + parser.add_argument("--out", type=Path, default=BACKEND_DIR.parent / "docs" / "scaling") + args = parser.parse_args() + + settings = Settings() + migrate(settings.database_url) + total = args.saturation + args.rate_events + events = list(itertools.islice(test_events(settings.dataset_dir), total)) + results = [run(settings, n, events, args.saturation, args.rate) for n in args.workers] + + load_avg = os.getloadavg() + summary = { + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "machine": { + "platform": platform.platform(), + "processor": platform.processor(), + "cpu_count": os.cpu_count(), + "load_average_at_end": [round(x, 1) for x in load_avg], + }, + "dataset": settings.dataset, + "events": total, + "settings": vars(args) | {"out": str(args.out)}, + "results": results, + } + args.out.mkdir(parents=True, exist_ok=True) + path = args.out / f"loadtest-{time.strftime('%Y%m%d-%H%M%S')}.json" + path.write_text(json.dumps(summary, indent=2)) + print(f"\nwritten {path}\n\n{table(results)}") + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/platform/models.py b/backend/src/fraudlens/platform/models.py index c69528a..8327b3d 100644 --- a/backend/src/fraudlens/platform/models.py +++ b/backend/src/fraudlens/platform/models.py @@ -39,6 +39,18 @@ VERDICTS = ("confirmed_fraud", "legitimate", "inconclusive") CASE_SOURCE = ("alert", "customer_report", "manual") FREEZE_STATUS = ("pending", "approved", "rejected") +APPEAL_STATUS = ("pending", "approved", "rejected") +# How the customer says they know the person they are paying. +APPEAL_RELATIONS = ( + "family", + "friend", + "business", + "seller", + "landlord", + "employer", + "other", + "none", +) def _one_of(column: str, values: tuple[str, ...]) -> str: @@ -231,6 +243,25 @@ class ShadowScore(Base): created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) +class PendingDecision(Base): + """A transaction that is in the state and still waits for its decision. + + A stream worker applies recorded events in order under the scorer's lock and + decides them after releasing it (PLATFORM §8). The feature vector is kept here, + committed with the transaction, so whichever worker finishes the work makes the + same decision; the row is deleted in the commit that writes the decision. + """ + + __tablename__ = "pending_decisions" + + txn_id: Mapped[int] = mapped_column( + BigInteger, ForeignKey("transactions.txn_id", ondelete="CASCADE"), primary_key=True + ) + features: Mapped[list[float]] = mapped_column(ARRAY(Float(53))) + context: Mapped[dict] = mapped_column(JSONB) + created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) + + class CaseEvent(Base): """The case timeline: who did what, in order. Rows are never changed.""" @@ -305,6 +336,38 @@ class CustomerReport(Base): created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) +class Appeal(Base): + """A customer says a warned or held payment is genuine. A person answers it.""" + + __tablename__ = "appeals" + __table_args__ = ( + CheckConstraint(_one_of("status", APPEAL_STATUS), name="status"), + CheckConstraint(_one_of("relation", APPEAL_RELATIONS), name="relation"), + CheckConstraint("tier IN ('warn', 'step_up', 'hold')", name="tier"), + CheckConstraint("(status = 'pending') = (decided_by IS NULL)", name="decided_has_decider"), + Index("ix_appeals_status_sla", "status", "sla_due_at"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + # One appeal per payment. + txn_id: Mapped[int] = mapped_column( + BigInteger, ForeignKey("transactions.txn_id", ondelete="CASCADE"), unique=True + ) + wallet_id: Mapped[str] = mapped_column(String(32), index=True) # the sender, who appeals + case_id: Mapped[int | None] = mapped_column(ForeignKey("cases.id")) + tier: Mapped[str] = mapped_column(String(8)) # the decision's tier when it was appealed + relation: Mapped[str] = mapped_column(String(16)) + # Free text typed by a customer: shown masked, never used to decide anything. + reason: Mapped[str] = mapped_column(Text) + status: Mapped[str] = mapped_column(String(12), server_default="pending") + filed_at: Mapped[datetime] = mapped_column(Timestamp) # domain time, like opened_at + sla_due_at: Mapped[datetime] = mapped_column(Timestamp) + decided_by: Mapped[int | None] = mapped_column(ForeignKey("users.id")) + decision_note: Mapped[str | None] = mapped_column(Text) + decided_at: Mapped[datetime | None] = mapped_column(Timestamp) + created_at: Mapped[datetime] = mapped_column(Timestamp, server_default=func.now()) + + class PastCase(Base): """Confirmed fraud cases from before the platform went live (the training period).""" diff --git a/backend/src/fraudlens/platform/scoring.py b/backend/src/fraudlens/platform/scoring.py index 10d6b3f..9d3509e 100644 --- a/backend/src/fraudlens/platform/scoring.py +++ b/backend/src/fraudlens/platform/scoring.py @@ -16,6 +16,14 @@ analyst releases it. Replayed transactions are recorded history, so they are applied on arrival (the model was trained on features computed that way) while the workflow still treats a hold as held. + +Several processes may score at once (stream workers, the API). Each one is a +replica: before it changes the state it takes the sequencing lock (a Postgres +advisory lock) and catches up with the log, so the state changes stay one at a +time and in order. That ordered step is microseconds; the decision is a +millisecond, and for recorded events and unscored types it does not change the +state, so a worker stores the feature vector in `pending_decisions`, releases +the lock and decides in parallel with the others (`sequence` then `finish`). """ from __future__ import annotations @@ -25,21 +33,24 @@ import math import threading import time -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timedelta +from typing import Any from redis import Redis -from sqlalchemy import func, insert, select +from sqlalchemy import delete, func, insert, select, text, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, sessionmaker from ..config import Settings from ..decision import Decision as EngineDecision from ..decision import DecisionEngine, SimilarCases, context_from_engine, load_policy +from ..decision.customer import cue as customer_cue from ..decision.policy import RANK from ..features import SCORED_TYPES, FeatureEngine, Txn +from ..intel.memory import MessageMemory, escalate from ..mlops.shadow import Shadow from ..models import registry from .audit import WorkflowError @@ -49,6 +60,7 @@ Case, CaseEvent, Decision, + PendingDecision, ShadowScore, Transaction, Wallet, @@ -61,6 +73,21 @@ # Events may arrive a little out of order; one this far behind the stream cannot be scored. STALE_SECONDS = 300.0 +# Transaction-scoped advisory locks shared by every process that writes. +SEQUENCE_LOCK = text("SELECT pg_advisory_xact_lock(1, 1)") # state changes, one at a time +CASES_LOCK = text("SELECT pg_advisory_xact_lock(1, 2)") # the workflow, against decisions +CASES_SHARED = text("SELECT pg_advisory_xact_lock_shared(1, 2)") # decisions, among themselves +SUBJECT_LOCK = text("SELECT pg_advisory_xact_lock(2, hashtext(:subject))") # one wallet's case +FROZEN_VERSION_SUFFIX = ":frozen-version" # bumped whenever a wallet is frozen or released +MULE_SUFFIX = ":mule-scores" # the highest mule score seen per wallet, across processes + +_LOG_COLUMNS = ( + Transaction.txn_id, Transaction.ts, Transaction.type, Transaction.sender_id, + Transaction.sender_type, Transaction.receiver_id, Transaction.receiver_type, + Transaction.amount, Transaction.sender_balance_before, Transaction.device_id, + Transaction.channel, Transaction.district, Transaction.network, +) # fmt: skip + class Clock: """Domain time: follows the event stream, and ticks in real time between events. @@ -97,9 +124,42 @@ class Result: alert: dict | None = None # published to the live feed +@dataclass +class Scored: + """A scored transaction that has been through the ordered step: its features + are fixed, its decision may still have to be made and recorded.""" + + result: Result + txn: Txn + source: str + features: Sequence[float] + context: dict[str, float] + when: datetime + elapsed: float = 0.0 # seconds spent on features and decision: the recorded latency + decision: EngineDecision | None = None + row: dict | None = None # the decision row, once decided + shadow: dict | None = None + link: dict | None = None # the flagged message this payment follows, if any + resumed: bool = False # left pending by a worker that stopped; already in the database + + +def subject_of(t: Txn) -> str: + """The wallet under suspicion: who receives a transfer, who initiates anything else.""" + return t.receiver_id if t.type == "SEND_MONEY" else t.sender_id + + +def _pack(context: dict[str, float]) -> dict[str, Any]: + # JSONB has no NaN or infinity; keep them as text so the decision can be remade exactly. + return {k: float(v) if math.isfinite(v) else repr(float(v)) for k, v in context.items()} + + +def _unpack(data: dict[str, Any]) -> dict[str, float]: + return {k: float(v) for k, v in data.items()} + + _ALERT_DETAIL = ( "rule_trace", "fallback_signals", "reasons", "customer_message", - "recommended_actions", "similar_cases", "evidence", + "recommended_actions", "similar_cases", "evidence", "segment", ) # fmt: skip @@ -155,14 +215,15 @@ def __init__( log.warning("no similar-case index for model %s", self.bundle.version) self.decision = DecisionEngine(self.policy, self.bundle, similar) self.shadow = self._load_shadow() - self._shadow_rows: list[dict] = [] self.engine = FeatureEngine() self.clock = Clock(0.0) self.frozen: set[str] = set() + self.messages = MessageMemory() # flagged messages a payment may follow self.mule_scores: dict[str, float] = {} # wallets the mule model has alerted on - self.seq = 0 # last position written to the log - self._new_wallets: list[dict] = [] - self._new_agents: list[dict] = [] + self.seq = 0 # last position of the log applied to this replica + self._frozen_key = settings.events_stream + FROZEN_VERSION_SUFFIX + self._mule_key = settings.events_stream + MULE_SUFFIX + self._frozen_seen: int | None = None self.ready = False # False while the in-memory state may not match the database self.recover() @@ -182,24 +243,25 @@ def _load_shadow(self) -> Shadow | None: log.exception("shadow model %s could not be loaded; shadow mode is off", version) return None - def _shadow_score(self, txn_id: int, features) -> None: - """Record what the challenger says. It can fail; the decision cannot depend on it.""" + def _shadow_score(self, txn_id: int, features) -> dict | None: + """What the challenger says. It can fail; the decision cannot depend on it.""" + shadow = self.shadow + if shadow is None: + return None t0 = time.perf_counter() try: - risk, tier = self.shadow.score(features) + risk, tier = shadow.score(features) except Exception: log.exception("shadow model failed on transaction %d; shadow mode is off", txn_id) self.shadow = None - return - self._shadow_rows.append( - { - "txn_id": txn_id, - "model_version": self.shadow.version, - "risk": risk, - "tier": tier, - "latency_ms": (time.perf_counter() - t0) * 1000, - } - ) + return None + return { + "txn_id": txn_id, + "model_version": shadow.version, + "risk": risk, + "tier": tier, + "latency_ms": (time.perf_counter() - t0) * 1000, + } # ------------------------------------------------------------- recovery @@ -207,14 +269,7 @@ def recover(self) -> None: """Rebuild the in-memory state from the baseline snapshot and the database.""" t0 = time.perf_counter() engine = FeatureEngine.load(self.settings.dataset_dir / BASELINE_SNAPSHOT) - columns = ( - Transaction.txn_id, Transaction.ts, Transaction.type, Transaction.sender_id, - Transaction.sender_type, Transaction.receiver_id, Transaction.receiver_type, - Transaction.amount, Transaction.sender_balance_before, Transaction.device_id, - Transaction.channel, Transaction.district, Transaction.network, - Transaction.applied_at, Transaction.applied_seq, - ) # fmt: skip - applied = seq = 0 + version = self._frozen_version() with self.sessions() as s: # Parties first seen after the baseline were registered in the database. for w in s.execute(select(Wallet.wallet_id, Wallet.created_at, Wallet.district)): @@ -222,28 +277,7 @@ def recover(self) -> None: engine.register_wallet(w.wallet_id, w.created_at.timestamp(), w.district) for a in s.execute(select(Agent.agent_id, Agent.district)): engine.register_agent(a.agent_id, a.district) - flags = s.execute( - select(WalletFlag.wallet_id, WalletFlag.flagged_at, WalletFlag.applied_seq) - .where(WalletFlag.applied_seq.is_not(None)) - .order_by(WalletFlag.applied_seq) - ).all() - j = 0 - rows = s.execute( - select(*columns) - .where(Transaction.applied_seq.is_not(None)) - .order_by(Transaction.applied_seq) - .execution_options(yield_per=20_000) - ) - for row in rows: - while j < len(flags) and flags[j].applied_seq < row.applied_seq: - engine.flag_wallet(flags[j].wallet_id, flags[j].flagged_at.timestamp()) - j += 1 - engine.update(txn_from_row(row, row.applied_at.timestamp())) - applied, seq = applied + 1, row.applied_seq - for flag in flags[j:]: - engine.flag_wallet(flag.wallet_id, flag.flagged_at.timestamp()) - if flags: - seq = max(seq, flags[-1].applied_seq) + seq, applied, flagged = self._replay(s, engine, 0) frozen = set(s.scalars(select(Wallet.wallet_id).where(Wallet.status == "frozen"))) mule_scores: dict[str, float] = {} if self.decision.mule_threshold is not None: @@ -258,15 +292,133 @@ def recover(self) -> None: ) with self.lock: self.engine, self.frozen, self.mule_scores = engine, frozen, mule_scores - self.seq = seq + self.seq, self._frozen_seen = seq, version self.ready = True self.clock = Clock(engine.last_ts) + self._merge_mule_scores() log.info( "scorer state rebuilt in %.1fs: %d transactions and %d flags applied since baseline", time.perf_counter() - t0, applied, - len(flags), + flagged, + ) + + def _replay(self, s: Session, engine: FeatureEngine, after: int) -> tuple[int, int, int]: + """Apply the log after position `after` to `engine`, in order. + Returns the last position, and how many transactions and flags were applied.""" + flags = s.execute( + select(WalletFlag.wallet_id, WalletFlag.flagged_at, WalletFlag.applied_seq) + .where(WalletFlag.applied_seq > after) + .order_by(WalletFlag.applied_seq) + ).all() + query = ( + select(*_LOG_COLUMNS, Transaction.applied_at, Transaction.applied_seq) + .where(Transaction.applied_seq > after) + .order_by(Transaction.applied_seq) ) + if after: + # Catching up: another process may have registered parties this one never saw. + rows = s.execute(query).all() + self._register_known(s, engine, rows, [f.wallet_id for f in flags]) + else: + rows = s.execute(query.execution_options(yield_per=20_000)) + seq, applied, j = after, 0, 0 + for row in rows: + while j < len(flags) and flags[j].applied_seq < row.applied_seq: + engine.flag_wallet(flags[j].wallet_id, flags[j].flagged_at.timestamp()) + j += 1 + engine.update(txn_from_row(row, row.applied_at.timestamp())) + applied, seq = applied + 1, row.applied_seq + for flag in flags[j:]: + engine.flag_wallet(flag.wallet_id, flag.flagged_at.timestamp()) + if flags: + seq = max(seq, flags[-1].applied_seq) + return seq, applied, len(flags) + + @staticmethod + def _register_known(s: Session, engine: FeatureEngine, txns, wallets=()) -> None: + """Register, from their database rows, the parties of `txns` (and `wallets`) that + are in the database but not yet in `engine`: first seen by another process.""" + missing_wallets, missing_agents = {w for w in wallets if w not in engine.wallets}, set() + for t in txns: + for party, kind in ((t.sender_id, t.sender_type), (t.receiver_id, t.receiver_type)): + if kind == "wallet" and party not in engine.wallets: + missing_wallets.add(party) + elif kind == "agent" and party not in engine.agents: + missing_agents.add(party) + if missing_wallets: + for w in s.execute( + select(Wallet.wallet_id, Wallet.created_at, Wallet.district).where( + Wallet.wallet_id.in_(missing_wallets) + ) + ): + engine.register_wallet(w.wallet_id, w.created_at.timestamp(), w.district) + if missing_agents: + for a in s.execute( + select(Agent.agent_id, Agent.district).where(Agent.agent_id.in_(missing_agents)) + ): + engine.register_agent(a.agent_id, a.district) + + def refresh(self, s: Session) -> int: + """Catch this replica up with what other processes have committed. The caller + holds `lock`, and the sequencing lock unless `s` reads one consistent snapshot. + Returns how many state changes were applied.""" + version = self._frozen_version() + if version is None or version != self._frozen_seen: + self.frozen = set(s.scalars(select(Wallet.wallet_id).where(Wallet.status == "frozen"))) + self._frozen_seen = version + seq, applied, flagged = self._replay(s, self.engine, self.seq) + if applied or flagged: + self.seq = seq + self.clock.observe(self.engine.last_ts) + return applied + flagged + + def follow(self, stop: threading.Event, interval: float = 0.5) -> None: + """Keep a replica that does not score itself (the API next to stream workers) + current for reads: the graph, the customer view, what-if.""" + while not stop.wait(interval): + try: + with self.lock: + if not self.ready: + self.recover() + continue + with self.sessions() as s: + s.connection(execution_options={"isolation_level": "REPEATABLE READ"}) + self.refresh(s) + self._merge_mule_scores() + except Exception: + self.ready = False # half a catch-up is not a state: rebuild next time + log.exception("could not follow the log; rebuilding") + + def _frozen_version(self) -> int | None: + """A counter bumped on every freeze and release: 0 without Redis (one process), + None when Redis cannot say (then the frozen set is re-read).""" + if self.redis is None: + return 0 + try: + return int(self.redis.get(self._frozen_key) or 0) + except Exception: + return None + + def _merge_mule_scores(self) -> None: + if self.redis is None or self.decision.mule_threshold is None: + return + try: + shared = self.redis.zrange(self._mule_key, 0, -1, withscores=True) + except Exception: + return # display only: the graph shows what this replica saw + for wallet, score in shared: + wallet = wallet.decode() if isinstance(wallet, bytes) else wallet + if score > self.mule_scores.get(wallet, 0.0): + self.mule_scores[wallet] = score + + def _share_mule_scores(self, scores: dict[str, float]) -> None: + if not scores or self.redis is None: + return + try: + self.redis.zadd(self._mule_key, scores, gt=True) + except Exception: + log.warning("could not share %d mule scores", len(scores)) # ------------------------------------------------------------ processing @@ -276,30 +428,86 @@ def process(self, events: list[TxnEvent | FlagEvent]) -> list[Result]: Idempotent on `txn_id`: an event seen before returns the recorded outcome and changes nothing, so a caller or the stream may safely send it again. """ - with self.lock: - if not self.ready: - self.recover() - try: - with self.sessions() as s: - results = self._process(s, events) - s.commit() - except Exception: - # The engine may be ahead of what was committed. Rebuild it from the log; - # if that fails too (database down), the next call tries again first. - log.exception("batch failed; rebuilding scorer state from the database") - self.ready = False - self.recover() - raise + mule: dict[str, float] = {} + with self.lock, self._ordered() as s: + results, scored = self._sequence(s, events) + self._decide(scored, mule) + self._record(s, scored) + for result in results: + if result.duplicate: + self._recorded(s, result) + s.commit() + self._share_mule_scores(mule) self._publish([r.alert for r in results if r.alert]) return results - def _process(self, s: Session, events: list[TxnEvent | FlagEvent]) -> list[Result]: + def sequence(self, events: list[TxnEvent | FlagEvent]) -> tuple[list[Result], list[Scored]]: + """A stream worker's ordered step: apply and record the batch, and leave the + decisions to make in `pending_decisions`, committed with it. Entries that a + stopped worker left half done come back with their stored features.""" + with self.lock, self._ordered() as s: + results, scored = self._sequence(s, events, resume=True) + fresh = [item for item in scored if not item.resumed] + if fresh: + s.execute( + insert(PendingDecision), + [ + { + "txn_id": item.txn.txn_id, + "features": [float(v) for v in item.features], + "context": _pack(item.context), + } + for item in fresh + ], + ) + s.commit() + return results, scored + + def finish(self, scored: list[Scored]) -> list[Scored]: + """A stream worker's parallel step: decide, then record each decision exactly + once (the pending row is claimed in the same commit). Returns what this call + recorded; an item another worker already finished is left out.""" + if not scored: + return [] + mule: dict[str, float] = {} + self._decide(scored, mule) + with self.sessions() as s: + done = self._record(s, scored, claim=True) + s.commit() + self._share_mule_scores(mule) + self._publish([item.result.alert for item in done if item.result.alert]) + return done + + @contextmanager + def _ordered(self) -> Iterator[Session]: + """A session holding the sequencing lock, with this replica caught up. + The caller holds `lock`.""" + if not self.ready: + self.recover() + try: + with self.sessions() as s: + s.execute(SEQUENCE_LOCK) + self.refresh(s) + yield s + except Exception: + # The engine may be ahead of what was committed. Rebuild it from the log; + # if that fails too (database down), the next call tries again first. + log.exception("batch failed; rebuilding scorer state from the database") + self.ready = False + self.recover() + raise + + def _sequence( + self, s: Session, events: list[TxnEvent | FlagEvent], resume: bool = False + ) -> tuple[list[Result], list[Scored]]: ids = [e.txn.txn_id for e in events if isinstance(e, TxnEvent)] seen = set(s.scalars(select(Transaction.txn_id).where(Transaction.txn_id.in_(ids)))) + self._register_known(s, self.engine, [e.txn for e in events if isinstance(e, TxnEvent)]) results: list[Result] = [] txn_rows: list[dict] = [] - self._new_wallets, self._new_agents, self._shadow_rows = [], [], [] - decided: list[tuple[Result, Txn, EngineDecision, dict]] = [] + new_wallets: list[dict] = [] + new_agents: list[dict] = [] + scored: list[Scored] = [] for event in events: if isinstance(event, FlagEvent): self.flag(s, event.wallet_id, event.ts, event.reason, event.source) @@ -314,35 +522,164 @@ def _process(self, s: Session, events: list[TxnEvent | FlagEvent]) -> list[Resul continue seen.add(t.txn_id) self.clock.observe(t.ts) - result, row, decision_row = self._one(event) + result, row, item = self._one(event, new_wallets, new_agents) txn_rows.append(row) results.append(result) - if decision_row is not None: - decided.append((result, t, *decision_row)) - - if self._new_wallets: - s.execute(pg_insert(Wallet).on_conflict_do_nothing(), self._new_wallets) - if self._new_agents: - s.execute(pg_insert(Agent).on_conflict_do_nothing(), self._new_agents) + if item is not None: + scored.append(item) + + duplicates = [r.txn_id for r in results if r.duplicate] + if resume and duplicates: + scored.extend(self._resume(s, duplicates)) + if new_wallets: + s.execute(pg_insert(Wallet).on_conflict_do_nothing(), new_wallets) + if new_agents: + s.execute(pg_insert(Agent).on_conflict_do_nothing(), new_agents) if txn_rows: s.execute(insert(Transaction), txn_rows) - decision_rows = [] - for result, t, decision, row in decided: - if decision.tier != "allow": - row["case_id"] = self._attach_to_case(s, t, decision.tier) - result.decision["case_id"] = result.alert["case_id"] = row["case_id"] - decision_rows.append(row) - if decision_rows: - s.flush() - s.execute(insert(Decision), decision_rows) - if self._shadow_rows: - s.execute(insert(ShadowScore), self._shadow_rows) - for result in results: - if result.duplicate: - self._recorded(s, result) - return results + return results, scored + + def _resume(self, s: Session, txn_ids: list[int]) -> list[Scored]: + """Transactions in the state whose decision was never recorded: the worker that + sequenced them stopped. The stored features make the same decision again.""" + rows = s.execute( + select( + *_LOG_COLUMNS, + Transaction.status, + Transaction.source, + PendingDecision.features, + PendingDecision.context, + ) + .join(PendingDecision, PendingDecision.txn_id == Transaction.txn_id) + .where(Transaction.txn_id.in_(txn_ids)) + ).all() + return [ + Scored( + result=Result(row.txn_id, row.status, scored=True), + txn=txn_from_row(row), + source=row.source, + features=list(row.features), + context=_unpack(row.context), + when=row.ts, + resumed=True, + ) + for row in rows + ] + + def _decision(self, item: Scored) -> EngineDecision: + decision = self.decision.decide(item.txn, item.features, item.context) + if item.link is not None: # paid what a flagged message asked + decision = escalate(decision, item.link, self.policy) + return decision + + def _decide(self, scored: list[Scored], mule: dict[str, float]) -> None: + """Make (or finish) the decisions. Reads no state that changes: safe outside + the lock and in parallel.""" + for item in scored: + t = item.txn + if item.decision is None: + t0 = time.perf_counter() + item.decision = self._decision(item) + item.elapsed += time.perf_counter() - t0 + if item.source == "replay": # applied already; the workflow still holds it + item.result.status = "held" if item.decision.tier == "hold" else "completed" + decision = item.decision + if decision.risk is not None: # after the clock stopped: not in latency_ms + item.shadow = self._shadow_score(t.txn_id, item.features) + score = decision.scores.get("mule") + if score is not None and score >= self.decision.mule_threshold: + if score > self.mule_scores.get(t.receiver_id, 0.0): + self.mule_scores[t.receiver_id] = score + mule[t.receiver_id] = max(score, mule.get(t.receiver_id, 0.0)) + item.row, item.result.alert = self._decision_row(item) + item.result.decision = self.summary(item.row) + + def _record(self, s: Session, scored: list[Scored], claim: bool = False) -> list[Scored]: + """Write the decisions, their cases and shadow scores. With `claim`, only the + ones whose pending row this commit deletes: two workers holding the same + entry write one decision.""" + if claim and scored: + pending = ( + select(PendingDecision.txn_id) + .where(PendingDecision.txn_id.in_([item.txn.txn_id for item in scored])) + .order_by(PendingDecision.txn_id) + .with_for_update() + ) + claimed = set( + s.scalars( + delete(PendingDecision) + .where(PendingDecision.txn_id.in_(pending.scalar_subquery())) + .returning(PendingDecision.txn_id) + ) + ) + scored = [item for item in scored if item.txn.txn_id in claimed] + if not scored: + return [] + alerting = [item for item in scored if item.decision.tier != "allow"] + if alerting: + # One open case per wallet: attach alerts on the same wallet one at a time. + s.execute(CASES_SHARED) + for subject in sorted({subject_of(item.txn) for item in alerting}): + s.execute(SUBJECT_LOCK, {"subject": subject}) + held = [ + item.txn.txn_id + for item in scored + if item.source == "replay" and item.result.status == "held" + ] + if held: + s.execute(update(Transaction).where(Transaction.txn_id.in_(held)).values(status="held")) + for item in alerting: + case_id = self._attach_to_case(s, item.txn, item.decision.tier) + item.row["case_id"] = item.result.decision["case_id"] = case_id + item.result.alert["case_id"] = case_id + s.flush() + s.execute(insert(Decision), [item.row for item in scored]) + shadow_rows = [item.shadow for item in scored if item.shadow is not None] + if shadow_rows: + s.execute(insert(ShadowScore), shadow_rows) + return scored + + def _decision_row(self, item: Scored) -> tuple[dict, dict | None]: + t, decision, when = item.txn, item.decision, item.when + tier, status = decision.tier, item.result.status + full = decision.to_dict() + alert = tier != "allow" + headline = None + if alert: + detail = {key: full[key] for key in _ALERT_DETAIL} + headline = next((r for r in decision.reasons if r["direction"] == "raises"), None) + detail["headline"] = headline and { + "en": headline["title_en"], + "bn": headline["title_bn"], + } + else: # an allowed transaction keeps only which rules it went through + trace = [{"id": r["id"], "status": r["status"]} for r in full["rule_trace"]] + detail = {"rule_trace": trace, "segment": full["segment"]} + decision_row = { + "txn_id": t.txn_id, "tier": tier, "action": decision.action, + "requires_review": decision.requires_review, "mode": decision.mode, + "decided_by": decision.decided_by, "model_tier": decision.model_tier, + "risk": decision.risk, "risk_score": decision.risk_score, + "risk_band": decision.risk_band, "scores": clean(decision.scores), + "model_version": decision.model_version, "policy_version": decision.policy_version, + "scenario": decision.scenario, "detail": clean(detail), + "narrative": decision.narrative, "features": [float(v) for v in item.features], + "latency_ms": item.elapsed * 1000, "case_id": None, "decided_at": when, + } # fmt: skip + if not alert: + return decision_row, None + return decision_row, { + "txn_id": t.txn_id, "ts": when.isoformat(), "type": t.type, "amount": t.amount, + "sender_id": t.sender_id, "receiver_id": t.receiver_id, "district": t.district, + "tier": tier, "risk_score": decision.risk_score, "decided_by": decision.decided_by, + "scenario": decision.scenario, "status": status, "case_id": None, + "headline_en": headline["title_en"] if headline else None, + "headline_bn": headline["title_bn"] if headline else None, + } # fmt: skip - def _one(self, event: TxnEvent) -> tuple[Result, dict, tuple[EngineDecision, dict] | None]: + def _one( + self, event: TxnEvent, new_wallets: list[dict], new_agents: list[dict] + ) -> tuple[Result, dict, Scored | None]: t, when = event.txn, moment(event.txn.ts) row = { "txn_id": t.txn_id, "ts": when, "type": t.type, @@ -360,7 +697,7 @@ def _one(self, event: TxnEvent) -> tuple[Result, dict, tuple[EngineDecision, dic # A frozen wallet is a decision two people made; the model is not consulted. row["status"], row["status_reason"] = "rejected", "wallet_frozen" return Result(t.txn_id, "rejected", status_reason="wallet_frozen"), row, None - self._register(t) + self._register(t, new_wallets, new_agents) if t.type not in SCORED_TYPES: self._apply(t) row["applied_at"], row["applied_seq"] = when, self.seq @@ -368,63 +705,36 @@ def _one(self, event: TxnEvent) -> tuple[Result, dict, tuple[EngineDecision, dic t0 = time.perf_counter() features = self.engine.features(t) - decision = self.decision.decide(t, features, context_from_engine(self.engine, t)) - latency_ms = (time.perf_counter() - t0) * 1000 - if self.shadow is not None and decision.risk is not None: - self._shadow_score(t.txn_id, features) # after the clock stopped: not in latency_ms + context = context_from_engine(self.engine, t) + result = Result(t.txn_id, "completed", scored=True) + item = Scored(result, t, event.source, features, context, when) + # Paid what a flagged message asked: read here, under the lock, with the clock. + item.link = self.messages.link(t.sender_id, t.receiver_id, t.amount, when) + if event.source == "replay": + # Recorded history: applied whatever the decision, which can wait for the lock + # to be released. The workflow's status follows in `_decide`. + item.elapsed = time.perf_counter() - t0 + self._apply(t) + row["applied_at"], row["applied_seq"] = when, self.seq + return item.result, row, item - tier = decision.tier + # Live: the decision says whether the state changes, so it is made here. + item.decision = self._decision(item) + item.elapsed = time.perf_counter() - t0 + tier = item.decision.tier if tier == "hold": status = "held" - elif tier == "allow" or event.source == "replay": - status = "completed" # in a replay the customer's choice is already history + elif tier == "allow": + status = "completed" else: status = "pending_customer" - if status == "completed" or event.source == "replay": + if status == "completed": self._apply(t) row["applied_at"], row["applied_seq"] = when, self.seq - row["status"] = status - mule = decision.scores.get("mule") - if mule is not None and mule >= self.decision.mule_threshold: - self.mule_scores[t.receiver_id] = max(mule, self.mule_scores.get(t.receiver_id, 0.0)) + row["status"] = item.result.status = status + return item.result, row, item - full = decision.to_dict() - alert = tier != "allow" - headline = None - if alert: - detail = {key: full[key] for key in _ALERT_DETAIL} - headline = next((r for r in decision.reasons if r["direction"] == "raises"), None) - detail["headline"] = headline and { - "en": headline["title_en"], - "bn": headline["title_bn"], - } - else: # an allowed transaction keeps only which rules it went through - trace = [{"id": r["id"], "status": r["status"]} for r in full["rule_trace"]] - detail = {"rule_trace": trace} - decision_row = { - "txn_id": t.txn_id, "tier": tier, "action": decision.action, - "requires_review": decision.requires_review, "mode": decision.mode, - "decided_by": decision.decided_by, "model_tier": decision.model_tier, - "risk": decision.risk, "risk_score": decision.risk_score, - "risk_band": decision.risk_band, "scores": clean(decision.scores), - "model_version": decision.model_version, "policy_version": decision.policy_version, - "scenario": decision.scenario, "detail": clean(detail), - "narrative": decision.narrative, "features": [float(v) for v in features], - "latency_ms": latency_ms, "case_id": None, "decided_at": when, - } # fmt: skip - result = Result(t.txn_id, status, scored=True, decision=self.summary(decision_row)) - if alert: - result.alert = { - "txn_id": t.txn_id, "ts": when.isoformat(), "type": t.type, "amount": t.amount, - "sender_id": t.sender_id, "receiver_id": t.receiver_id, "district": t.district, - "tier": tier, "risk_score": decision.risk_score, "decided_by": decision.decided_by, - "scenario": decision.scenario, "status": status, "case_id": None, - "headline_en": headline["title_en"] if headline else None, - "headline_bn": headline["title_bn"] if headline else None, - } # fmt: skip - return result, row, (decision, decision_row) - - def _register(self, t: Txn) -> None: + def _register(self, t: Txn, new_wallets: list[dict], new_agents: list[dict]) -> None: """A wallet or agent with no master record is registered on first sight, in the database too, so that a restart rebuilds exactly the same state.""" engine = self.engine @@ -432,7 +742,7 @@ def _register(self, t: Txn) -> None: if kind == "wallet" and party not in engine.wallets: when = moment(t.ts) engine.register_wallet(party, when.timestamp(), t.district) - self._new_wallets.append( + new_wallets.append( { "wallet_id": party, "created_at": when, @@ -444,7 +754,7 @@ def _register(self, t: Txn) -> None: ) elif kind == "agent" and party not in engine.agents: engine.register_agent(party, t.district) - self._new_agents.append({"agent_id": party, "district": t.district}) + new_agents.append({"agent_id": party, "district": t.district}) def _apply(self, t: Txn) -> None: self.engine.update(t) @@ -468,6 +778,9 @@ def summary(self, row) -> dict: "review_sla_minutes": spec.review_sla_minutes, "case_id": get("case_id"), "latency_ms": round(get("latency_ms"), 2), + # Which scam pattern the app's warning should ask about; see decision/customer.py. + "scenario": get("scenario"), + "cue": customer_cue(get("tier"), get("scenario"), get("detail")), } def _recorded(self, s: Session, result: Result) -> None: @@ -480,7 +793,7 @@ def _recorded(self, s: Session, result: Result) -> None: def _attach_to_case(self, s: Session, t: Txn, tier: str) -> int | None: """Holds open a case on the wallet under suspicion; lesser alerts join an open one.""" - subject = t.receiver_id if t.type == "SEND_MONEY" else t.sender_id + subject = subject_of(t) when = moment(t.ts) sla = self.policy.tiers["hold"].review_sla_minutes due = when + timedelta(minutes=sla) if sla and tier == "hold" else None @@ -527,7 +840,15 @@ def transaction(self) -> Iterator[Session]: self.recover() with self.sessions() as s: try: + # Before any decision being recorded elsewhere: cases change here too. + s.execute(SEQUENCE_LOCK) + s.execute(CASES_LOCK) + self.refresh(s) + frozen = set(self.frozen) yield s + if self.frozen != frozen and self.redis is not None: + # Other replicas re-read the frozen set before they score again. + self._frozen_seen = int(self.redis.incr(self._frozen_key)) s.commit() except WorkflowError: s.rollback() diff --git a/backend/src/fraudlens/platform/security.py b/backend/src/fraudlens/platform/security.py index 346fa57..b61fb9d 100644 --- a/backend/src/fraudlens/platform/security.py +++ b/backend/src/fraudlens/platform/security.py @@ -1,7 +1,9 @@ -"""Passwords, access tokens, rate limits and the checks that guard production.""" +"""Passwords, access tokens and their keys, client addresses, rate limits, production checks.""" from __future__ import annotations +import functools +import ipaddress import time import uuid from datetime import UTC, datetime, timedelta @@ -13,8 +15,11 @@ from redis import Redis from ..config import DEV_JWT_SECRET, Settings +from .keys import Key, Keyring, load_keyring ALGORITHM = "HS256" +DEFAULT_KID = "default" # the key id of FRAUDLENS_JWT_SECRET when there is no keyring +MIN_KEY_LENGTH = 32 ISSUER = "fraudlens" MIN_PASSWORD_LENGTH = 12 @@ -41,7 +46,18 @@ def verify_password(password_hash: str | None, password: str) -> bool: return False +def _jwt_keyring(settings: Settings) -> Keyring: + """The keyring file when there is one; otherwise the single secret, as key `default`.""" + if settings.jwt_keyring is not None: + return load_keyring(settings.jwt_keyring) + secret = settings.jwt_secret.get_secret_value() + return Keyring({DEFAULT_KID: Key(DEFAULT_KID, secret, datetime(1970, 1, 1, tzinfo=UTC))}) + + def issue_token(user_id: int, role: str, settings: Settings, now: datetime | None = None) -> str: + key = _jwt_keyring(settings).signing() + if key is None: + raise RuntimeError("the JWT keyring has no active key: run `keys rotate-jwt`") now = now or datetime.now(UTC) claims = { "sub": str(user_id), @@ -51,14 +67,19 @@ def issue_token(user_id: int, role: str, settings: Settings, now: datetime | Non "exp": now + timedelta(minutes=settings.jwt_ttl_minutes), "jti": uuid.uuid4().hex, # names this one token, so signing out can revoke it } - return jwt.encode(claims, settings.jwt_secret.get_secret_value(), algorithm=ALGORITHM) + # `kid` names the key, so a token outlives a rotation until it expires. + return jwt.encode(claims, key.secret, algorithm=ALGORITHM, headers={"kid": key.kid}) def read_token(token: str, settings: Settings) -> dict: try: + kid = jwt.get_unverified_header(token).get("kid", DEFAULT_KID) + key = _jwt_keyring(settings).verifying(kid) if isinstance(kid, str) else None + if key is None: + raise TokenError("signed with an unknown or retired key") return jwt.decode( token, - settings.jwt_secret.get_secret_value(), + key.secret, algorithms=[ALGORITHM], # never trust the algorithm named in the token issuer=ISSUER, options={"require": ["exp", "iat", "sub", "iss", "jti"]}, @@ -106,13 +127,77 @@ def reset(self, key: str) -> None: self.redis.delete(self._key(key)) +Network = ipaddress.IPv4Network | ipaddress.IPv6Network + + +@functools.cache +def _proxies(entries: tuple[str, ...]) -> tuple[Network, ...]: + nets = tuple(ipaddress.ip_network(e.strip(), strict=False) for e in entries) + if any(net.prefixlen == 0 for net in nets): + raise ValueError("FRAUDLENS_TRUSTED_PROXIES must not trust every address") + return nets + + +def proxies(entries: list[str]) -> tuple[Network, ...]: + return _proxies(tuple(entries)) + + +def _parse_ip(text: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: + try: + ip = ipaddress.ip_address(text.strip()) + except ValueError: + return None + if ip.version == 6 and ip.ipv4_mapped is not None: + ip = ip.ipv4_mapped + return ip + + +def client_address(peer: str | None, forwarded_for: list[str], trusted: list[str]) -> str | None: + """The address a request came from. `X-Forwarded-For` is believed only as far + back as it was written by trusted proxies: walking from the nearest hop, the + first address that is not one of ours is the client. Anything to the left of it + was written by the client and could say anything.""" + nets = proxies(trusted) + if peer is None or not nets: + return peer + hop = _parse_ip(peer) + if hop is None or not any(hop in net for net in nets): + return peer # not from our proxy: its header means nothing + chain = [part for header in forwarded_for for part in header.split(",") if part.strip()] + for part in reversed(chain): + ip = _parse_ip(part) + if ip is None: + break # a malformed hop: stop at the last address we could vouch for + hop = ip + if not any(ip in net for net in nets): + break + return str(hop) + + def check_production(settings: Settings) -> None: """Refuse to run a production service on development defaults.""" + proxies(settings.trusted_proxies) # malformed or catch-all entries fail everywhere if not settings.production: return - secret = settings.jwt_secret.get_secret_value() - if secret == DEV_JWT_SECRET or len(secret) < 32: - raise RuntimeError("FRAUDLENS_JWT_SECRET must be a private key of 32+ characters") + if settings.jwt_keyring is None: + secret = settings.jwt_secret.get_secret_value() + if secret == DEV_JWT_SECRET or len(secret) < MIN_KEY_LENGTH: + raise RuntimeError("FRAUDLENS_JWT_SECRET must be a private key of 32+ characters") + else: + ring = load_keyring(settings.jwt_keyring) + if ring.signing() is None: + raise RuntimeError("FRAUDLENS_JWT_KEYRING has no active key") + if any(k.secret == DEV_JWT_SECRET or len(k.secret) < MIN_KEY_LENGTH + for k in ring.keys.values()): # fmt: skip + raise RuntimeError("every key in FRAUDLENS_JWT_KEYRING must be 32+ characters") + if settings.ingest_keyring is not None: + ring = load_keyring(settings.ingest_keyring) + if any(len(k.secret) < MIN_KEY_LENGTH for k in ring.keys.values()): + raise RuntimeError("every key in FRAUDLENS_INGEST_KEYRING must be 32+ characters") + for partner in ring.partners: + url = ring.callback_url(partner) + if url and urlsplit(url).scheme != "https": + raise RuntimeError(f"the callback for {partner} must use https") if "fraudlens_dev" in settings.database_url: raise RuntimeError("FRAUDLENS_DATABASE_URL still uses the development password") if not urlsplit(settings.redis_url).password: diff --git a/backend/src/fraudlens/platform/stream.py b/backend/src/fraudlens/platform/stream.py index fa925c5..0b189b0 100644 --- a/backend/src/fraudlens/platform/stream.py +++ b/backend/src/fraudlens/platform/stream.py @@ -1,16 +1,34 @@ """The event stream: transactions in through a Redis stream, alerts out to a live feed. -Delivery is at least once. An entry is acknowledged only after its batch has +Delivery is at least once. An entry is acknowledged only after its decision has been committed, and scoring is idempotent on the transaction id, so a crash between the two means the entry is read again and recognised as a duplicate. An event that cannot be processed (malformed, too far behind the stream) goes to a dead-letter stream with the reason; it never blocks the events behind it. + +Any number of workers share one consumer group (docs/SCALING.md). Each batch +goes through two steps: + +1. In stream order, one batch at a time across all workers: the state changes + and the transaction rows (`Scorer.sequence`). A batch starts only when the + entry before it has been through this step: the cursor `:sequenced` + holds the last entry that has. +2. In parallel: the decisions, cases and alerts (`Scorer.finish`). Then the + entries are acknowledged and deleted, so the stream's length is the work + still to do, which is what the autoscaler watches. + +Entries a stopped worker held are taken over with XAUTOCLAIM once idle for +`claim_idle_ms`, by any worker, including one waiting for them. """ from __future__ import annotations import logging +import os +import socket import threading +import time +from collections import deque from collections.abc import AsyncIterator, Sequence from pydantic import BaseModel, ValidationError @@ -20,18 +38,46 @@ from ..config import Settings from .events import event_adapter -from .scoring import Scorer +from .scoring import FROZEN_VERSION_SUFFIX, MULE_SUFFIX, Result, Scored, Scorer log = logging.getLogger(__name__) GROUP = "scorer" -CONSUMER = "scorer-1" # one scorer owns the state, so there is one consumer DEAD_SUFFIX = ":dead" +CURSOR_SUFFIX = ":sequenced" +# Keys that describe the stream's contents, deleted with it when the database is reset. +RESET_SUFFIXES = ("", DEAD_SUFFIX, CURSOR_SUFFIX, FROZEN_VERSION_SUFFIX, MULE_SUFFIX) DEAD_MAXLEN = 10_000 MAX_BACKLOG = 200_000 +LATENCY_SAMPLES = 500_000 # The database or Redis is unavailable: leave the entries pending and try again. RETRYABLE = (OperationalError, InterfaceError, RedisError) +# The cursor only moves forward: entry ids compare as (milliseconds, sequence). +_ADVANCE = """ +local cur = redis.call('GET', KEYS[1]) +if cur then + local cm, cs = string.match(cur, '(%d+)-(%d+)') + local nm, ns = string.match(ARGV[1], '(%d+)-(%d+)') + cm, cs, nm, ns = tonumber(cm), tonumber(cs), tonumber(nm), tonumber(ns) + if cm > nm or (cm == nm and cs >= ns) then return 0 end +end +redis.call('SET', KEYS[1], ARGV[1]) +return 1 +""" + +Entry = tuple[str, dict] + + +def entry_key(entry_id: str) -> tuple[int, int]: + ms, _, seq = entry_id.partition("-") + return int(ms), int(seq or 0) + + +def default_consumer() -> str: + """Unique per process; stable across restarts of a container (pod name, pid 1).""" + return f"{socket.gethostname()}-{os.getpid()}" + def enqueue(redis: Redis, stream: str, events: Sequence[BaseModel]) -> int: pipe = redis.pipeline(transaction=False) @@ -43,16 +89,33 @@ def enqueue(redis: Redis, stream: str, events: Sequence[BaseModel]) -> int: class Worker(threading.Thread): def __init__( - self, scorer: Scorer, redis: Redis, settings: Settings, batch: int = 200, block_ms=1000 + self, + scorer: Scorer, + redis: Redis, + settings: Settings, + batch: int = 200, + block_ms=1000, + consumer: str | None = None, ) -> None: super().__init__(name="fraudlens-stream-worker", daemon=True) self.scorer, self.redis = scorer, redis self.stream = settings.events_stream self.dead = settings.events_stream + DEAD_SUFFIX + self.cursor = settings.events_stream + CURSOR_SUFFIX + self.consumer = consumer or settings.worker_name or default_consumer() + self.claim_idle_ms = settings.claim_idle_ms self.batch, self.block_ms = batch, block_ms - self.processed = self.dead_lettered = self.failures = 0 + self.processed = self.dead_lettered = self.failures = self.reclaimed = 0 + self.decided = 0 # decisions this worker recorded + # (when recorded, seconds since the entry was added): end-to-end latency + self.latencies: deque[tuple[float, float]] = deque(maxlen=LATENCY_SAMPLES) self._stopping = threading.Event() self._caught_up = False # entries delivered before a crash are handled first + # Entries this consumer holds and has not finished, in stream order. Each unit is + # contiguous in the stream (a fresh read), or a single entry (pending, reclaimed). + self._units: list[list[Entry]] = [] + self._last_claim = 0.0 + self._advance = redis.register_script(_ADVANCE) def ensure_group(self) -> None: try: @@ -64,17 +127,84 @@ def ensure_group(self) -> None: def backlog(self) -> int: return int(self.redis.xlen(self.stream)) - def _read(self, block_ms: int | None) -> list[tuple[str, dict]]: + # ---------------------------------------------------------------- reading + + def _held(self) -> set[str]: + return {entry_id for unit in self._units for entry_id, _ in unit} + + def _add(self, units: list[list[Entry]]) -> None: + held = self._held() + self._units.extend(u for u in units if u and u[0][0] not in held) + self._units.sort(key=lambda u: entry_key(u[0][0])) + + def _fill(self, block_ms: int | None) -> None: + if self._units: + return if not self._caught_up: - reply = self.redis.xreadgroup(GROUP, CONSUMER, {self.stream: "0"}, count=self.batch) + reply = self.redis.xreadgroup( + GROUP, self.consumer, {self.stream: "0"}, count=self.batch + ) entries = reply[0][1] if reply else [] if entries: - return entries + self._add([[e] for e in entries]) + return self._caught_up = True + self._claim() + if self._units: + return reply = self.redis.xreadgroup( - GROUP, CONSUMER, {self.stream: ">"}, count=self.batch, block=block_ms + GROUP, self.consumer, {self.stream: ">"}, count=self.batch, block=block_ms ) - return reply[0][1] if reply else [] + if reply and reply[0][1]: + self._add([reply[0][1]]) + + def _claim(self, force: bool = False) -> None: + """Take over entries another consumer has held too long (it stopped).""" + now = time.monotonic() + if not force and now - self._last_claim < self.claim_idle_ms / 2000: + return + self._last_claim = now + start, claimed = "0-0", [] + while True: + reply = self.redis.xautoclaim( + self.stream, GROUP, self.consumer, self.claim_idle_ms, start, count=self.batch + ) + start, entries = reply[0], reply[1] + claimed.extend(e for e in entries if e[1] is not None) + if start in ("0-0", b"0-0") or not entries: + break + held = self._held() + fresh = [e for e in claimed if e[0] not in held] + if fresh: + self.reclaimed += len(fresh) + log.warning("%s took over %d idle entries", self.consumer, len(fresh)) + self._add([[e] for e in fresh]) + + # ------------------------------------------------------------- ordering + + def _sequenced_before(self, entry_id: str) -> bool: + """Whether every entry before this one has been through the ordered step.""" + before = self.redis.xrevrange(self.stream, max="(" + entry_id, min="-", count=1) + if not before: + return True + cursor = self.redis.get(self.cursor) + return cursor is not None and entry_key(before[0][0]) <= entry_key(cursor) + + def _next_unit(self) -> list[Entry] | None: + """The earliest unit held, once it may be sequenced. While waiting, entries + held by a stopped consumer are taken over and done first.""" + delay = 0.001 + while not self._stopping.is_set(): + unit = self._units[0] + if self._sequenced_before(unit[0][0]): + return unit + self._claim() + if self._units[0] is unit: + time.sleep(delay) + delay = min(delay * 2, 0.02) + return None + + # ------------------------------------------------------------- handling def _bury(self, pipe, entry_id: str, fields: dict, reason: str) -> None: self.dead_lettered += 1 @@ -87,10 +217,19 @@ def _bury(self, pipe, entry_id: str, fields: dict, reason: str) -> None: ) def run_once(self, block_ms: int | None = None) -> int: - """Read one batch, score it, acknowledge it. Returns the number of entries handled.""" - entries = self._read(block_ms) - if not entries: + """Read (or take over) one batch, score it, acknowledge it. Returns the number + of entries handled.""" + self._fill(block_ms) + if not self._units: + return 0 + unit = self._next_unit() + if unit is None: return 0 + handled = self._handle(unit) + self._units.remove(unit) + return handled + + def _handle(self, entries: list[Entry]) -> int: pipe = self.redis.pipeline(transaction=False) ids, events = [], [] for entry_id, fields in entries: @@ -108,23 +247,10 @@ def run_once(self, block_ms: int | None = None) -> int: ids.append((entry_id, fields)) events.append(event) - try: - results = self.scorer.process(events) if events else [] - except RETRYABLE: - raise - except Exception: - # One bad event must not take the batch with it: go through them singly. - log.exception("batch of %d failed; processing its events one by one", len(events)) - results = [] - for (entry_id, fields), event in zip(ids, events, strict=True): - try: - results.extend(self.scorer.process([event])) - except RETRYABLE: - raise - except Exception as exc: - self.failures += 1 - self._bury(pipe, entry_id, fields, f"processing failed: {type(exc).__name__}") - results.append(None) + results, scored = self._sequence(pipe, ids, events) + # Everything up to the last entry of this unit is in order now. + self._advance(keys=[self.cursor], args=[entries[-1][0]]) + self._finish(pipe, ids, results, scored) for (entry_id, fields), result in zip(ids, results, strict=True): if result is not None and result.status == "stale": self._bury(pipe, entry_id, fields, "stale: too far behind the stream") @@ -132,9 +258,59 @@ def run_once(self, block_ms: int | None = None) -> int: pipe.xack(self.stream, GROUP, *done) pipe.xdel(self.stream, *done) pipe.execute() + now = time.time() + self.latencies.extend((now, now - entry_key(e)[0] / 1000) for e in done) self.processed += len(done) return len(done) + def _sequence(self, pipe, ids, events) -> tuple[list[Result | None], list[Scored]]: + if not events: + return [], [] + try: + return self.scorer.sequence(events) + except RETRYABLE: + raise + except Exception: + # One bad event must not take the batch with it: go through them singly. + log.exception("batch of %d failed; processing its events one by one", len(events)) + results: list[Result | None] = [] + scored: list[Scored] = [] + for (entry_id, fields), event in zip(ids, events, strict=True): + try: + one, items = self.scorer.sequence([event]) + except RETRYABLE: + raise + except Exception as exc: + self.failures += 1 + self._bury(pipe, entry_id, fields, f"processing failed: {type(exc).__name__}") + results.append(None) + continue + results.extend(one) + scored.extend(items) + return results, scored + + def _finish(self, pipe, ids, results, scored: list[Scored]) -> None: + try: + self.decided += len(self.scorer.finish(scored)) + return + except RETRYABLE: + raise + except Exception: + log.exception("decisions for %d events failed; recording them one by one", len(scored)) + entry_of = {r.txn_id: entry for entry, r in zip(ids, results, strict=True) if r} + for item in scored: + try: + self.decided += len(self.scorer.finish([item])) + except RETRYABLE: + raise + except Exception as exc: + # The transaction stays applied, with its pending row to show for it. + self.failures += 1 + entry_id, fields = entry_of.get(item.txn.txn_id, ("?", {})) + self._bury(pipe, entry_id, fields, f"decision failed: {type(exc).__name__}") + + # ------------------------------------------------------------ lifecycle + def run(self) -> None: while not self._stopping.is_set(): try: @@ -144,6 +320,7 @@ def run(self) -> None: except Exception: self.failures += 1 self._caught_up = False # whatever was delivered is still pending + self._units.clear() log.exception("stream worker error; retrying in 2s") self._stopping.wait(2.0) @@ -159,6 +336,30 @@ def drain(self) -> int: total += handled return total + def metrics(self, since: float | None = None, raw: bool = False) -> dict: + """Counters and end-to-end latency (entry added to decision recorded), over the + samples recorded after `since` (epoch seconds).""" + samples = sorted(lat for at, lat in list(self.latencies) if since is None or at >= since) + out = { + "consumer": self.consumer, + "processed": self.processed, + "decided": self.decided, + "dead_lettered": self.dead_lettered, + "failures": self.failures, + "reclaimed": self.reclaimed, + "latency_ms": { + f"p{q}": round( + samples[min(len(samples) - 1, int(len(samples) * q / 100))] * 1000, 2 + ) + for q in (50, 95, 99) + } + if samples + else {}, + } + if raw: + out["samples"] = [[at, lat] for at, lat in list(self.latencies)] + return out + async def alert_feed(redis, channel: str, heartbeat: float = 15.0) -> AsyncIterator[str]: """Server-sent events: one `alert` event per alert, and a comment line as a keep-alive. diff --git a/backend/src/fraudlens/platform/worker.py b/backend/src/fraudlens/platform/worker.py new file mode 100644 index 0000000..06e171f --- /dev/null +++ b/backend/src/fraudlens/platform/worker.py @@ -0,0 +1,103 @@ +"""One stream worker as its own process: run as many as the load needs. + + uv run python -m fraudlens.platform.worker [--port 8081] [--name W] + +It loads the model and policy once, rebuilds the feature state from the database +like the API does, and consumes the event stream in the `scorer` group next to +the other workers (docs/SCALING.md). A small HTTP port serves what an +orchestrator needs: + + GET /health the process is up (liveness) + GET /ready the state is rebuilt and the consumer is running (readiness) + GET /metrics counters and end-to-end latency, JSON; ?since=, ?raw=1 + +SIGTERM stops it after the batch in hand; what it has not acknowledged is taken +over by another worker after `FRAUDLENS_CLAIM_IDLE_MS`. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import signal +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + +from redis import Redis + +from ..config import Settings +from .db import make_engine, make_sessions +from .scoring import Scorer +from .stream import Worker + +log = logging.getLogger("fraudlens.worker") + + +def serve_health(worker: Worker, port: int) -> ThreadingHTTPServer: + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 (the stdlib's name) + url = urlparse(self.path) + query = parse_qs(url.query) + if url.path == "/health": + code, body = 200, {"status": "ok"} + elif url.path == "/ready": + ready = worker.scorer.ready and worker.is_alive() + code, body = (200 if ready else 503), {"ready": ready} + elif url.path == "/metrics": + since = float(query["since"][0]) if "since" in query else None + code = 200 + body = worker.metrics(since=since, raw=query.get("raw") == ["1"]) + else: + code, body = 404, {"error": "not_found"} + data = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args) -> None: # probes every few seconds: not worth a line + pass + + server = ThreadingHTTPServer(("0.0.0.0", port), Handler) + threading.Thread(target=server.serve_forever, name="fraudlens-health", daemon=True).start() + return server + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--port", type=int, help="health and metrics port") + parser.add_argument("--name", help="consumer name in the group (default: host-pid)") + parser.add_argument("--batch", type=int, default=200, help="entries read at a time") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + settings = Settings() + + db = make_engine(settings.database_url) + redis = Redis.from_url(settings.redis_url, decode_responses=True) + scorer = Scorer(settings, make_sessions(db), redis) + worker = Worker(scorer, redis, settings, batch=args.batch, consumer=args.name) + server = serve_health(worker, args.port or settings.worker_port) + + stopped = threading.Event() + + def stop(signum, _frame) -> None: + log.info("signal %d: stopping after the batch in hand", signum) + worker.stop() + stopped.set() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + worker.start() + log.info("worker %s consuming %s in %s mode", worker.consumer, worker.stream, scorer.mode) + stopped.wait() + worker.join(timeout=30.0) + server.shutdown() + redis.close() + db.dispose() + + +if __name__ == "__main__": + main() diff --git a/backend/src/fraudlens/secret_sources.py b/backend/src/fraudlens/secret_sources.py new file mode 100644 index 0000000..62fb201 --- /dev/null +++ b/backend/src/fraudlens/secret_sources.py @@ -0,0 +1,94 @@ +"""Settings from files and from a secrets store, so a secret never has to sit in the environment. + +Two sources, after the environment and before `.env`: + +- **Files.** `FRAUDLENS__FILE=/run/secrets/jwt_secret` reads `` from that + file (one trailing newline removed). This is the Docker and Kubernetes secrets + convention: the orchestrator mounts the file, the environment only names it. + Setting both `FRAUDLENS_` and `FRAUDLENS__FILE` is refused. +- **A provider.** `FRAUDLENS_SECRETS_PROVIDER=package.module:function` is called once + with the names of the settings still unset and returns a mapping of the ones it + knows, e.g. a function that reads Vault or AWS Secrets Manager. Nothing here + depends on any of those clients; the function brings its own. +""" + +from __future__ import annotations + +import importlib +import os +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +from pydantic.fields import FieldInfo +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource + +PREFIX = "FRAUDLENS_" +PROVIDER_VAR = PREFIX + "SECRETS_PROVIDER" +MAX_SECRET_BYTES = 64_000 + +Provider = Callable[[list[str]], Mapping[str, str]] + + +def read_secret_file(path: str | Path) -> str: + data = Path(path).read_bytes() + if len(data) > MAX_SECRET_BYTES: + raise ValueError(f"{path} is larger than a secret should be") + text = data.decode() + return text[:-1] if text.endswith("\n") else text + + +def load_provider(spec: str) -> Provider: + module, _, name = spec.partition(":") + if not module or not name: + raise ValueError(f"{PROVIDER_VAR} must look like 'package.module:function', not {spec!r}") + provider = getattr(importlib.import_module(module), name) + if not callable(provider): + raise TypeError(f"{spec} is not callable") + return provider + + +class SecretFilesSource(PydanticBaseSettingsSource): + """`FRAUDLENS__FILE` and the optional provider, for every setting.""" + + def __init__(self, settings_cls: type[BaseSettings], environ: Mapping[str, str] | None = None): + super().__init__(settings_cls) + self.environ = os.environ if environ is None else environ + + def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]: + return None, field_name, False # everything is done in __call__ + + def _value(self, name: str, field: FieldInfo, raw: str) -> Any: + return self.prepare_field_value(name, field, raw, self.field_is_complex(field)) + + def __call__(self) -> dict[str, Any]: + values: dict[str, Any] = {} + fields = self.settings_cls.model_fields + for name, field in fields.items(): + path = self.environ.get(f"{PREFIX}{name.upper()}_FILE") + if not path: + continue + if f"{PREFIX}{name.upper()}" in self.environ: + raise ValueError(f"set {PREFIX}{name.upper()} or its _FILE variant, not both") + values[name] = self._value(name, field, read_secret_file(path)) + spec = self.environ.get(PROVIDER_VAR) + if spec: + unset = [ + n for n in fields if n not in values and f"{PREFIX}{n.upper()}" not in self.environ + ] + for name, raw in load_provider(spec)(unset).items(): + if name in unset and raw is not None: + values[name] = self._value(name, fields[name], str(raw)) + return values + + +def directory_provider(names: list[str]) -> dict[str, str]: + """An example provider: one file per setting in `FRAUDLENS_SECRETS_DIR` (default + /run/secrets), named after the setting. Swap in a function that asks your vault.""" + root = Path(os.environ.get(PREFIX + "SECRETS_DIR", "/run/secrets")) + found = {} + for name in names: + path = root / name + if path.is_file(): + found[name] = read_secret_file(path) + return found diff --git a/backend/src/fraudlens/simulator/adversary.py b/backend/src/fraudlens/simulator/adversary.py new file mode 100644 index 0000000..9de120a --- /dev/null +++ b/backend/src/fraudlens/simulator/adversary.py @@ -0,0 +1,390 @@ +"""Adaptive adversaries: scam cells that change their scripts after FraudLens is deployed. + +Opt-in only. The default dataset never touches this module: the experiment +(`fraudlens.models.adversary`) runs the default world unchanged, then hands its +fraud planner to an `AdaptivePlanner` on the deployment day. From then on new +cells run in rounds; every scam draws one evasion tactic from the cells' shared +mix, and after each round the `Adversary` moves the mix toward the tactics whose +scams got past the deployed model. + +The tactics are the ones fraud teams report seeing once controls go live: + +- none: the original script +- split_amounts: each victim transfer split into a few payments below a cap +- seasoned_mule: an old customer wallet with years of normal history, bought or + rented, instead of a wallet opened for the purpose +- delayed_cashout: the mule waits hours to days, well past the 30-minute hold + review window, before forwarding or cashing out +- fan_out: a larger pool of mules, each victim paying the least-used one, so no + wallet collects many strangers' money +- mimic_hours: the scam call placed at the victim's own usual transacting hour +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass + +from .config import DAY +from .engine import CASH_OUT, SEND, VICTIM_ROLES, Intent +from .fraud import BASE_TYPOLOGIES, RECRUITABLE, WARMUP_DAYS, Case, Cell, FraudPlanner, Mule +from .world import DISTRICTS + +TACTICS = ("none", "split_amounts", "seasoned_mule", "delayed_cashout", "fan_out", "mimic_hours") +SPLIT_CAPS = (1000.0, 1500.0, 2000.0, 2500.0) # what a split payment is kept below +SEASONED_MIN_AGE_DAYS = 180 +DELAY_HOURS = (6.0, 48.0) +FAN_OUT_POOL = 8 # mules kept ready for fan-out scams, on top of the cell's usual three +SEASONED_POOL = 2 + + +@dataclass(frozen=True) +class AdversaryConfig: + """How the adversary behaves. The numbers are assumptions, stated in the model card.""" + + rounds: int = 6 + round_days: int = 7 + warmup_days: int = WARMUP_DAYS # new cells open and warm up mules before round 0 + n_cells: int = 8 + initial_none: float = 0.5 # round 0: half the scams run the old script + learning_rate: float = 3.0 # multiplicative-weights step on the success rate + floor: float = 0.03 # every tactic keeps being tried a little + min_cases: int = 5 # fewer scams than this with a tactic say nothing about it + adapt: bool = True # False keeps the round-0 mix: the non-adaptive control + seed: int = 11 + + +class Adversary: + """The cells' shared tactic mix and how it learns from the deployed model.""" + + def __init__(self, cfg: AdversaryConfig) -> None: + self.cfg = cfg + self.rng = random.Random(cfg.seed) + rest = (1.0 - cfg.initial_none) / (len(TACTICS) - 1) + self.mix = {t: (cfg.initial_none if t == "none" else rest) for t in TACTICS} + + def draw(self) -> str: + return self.rng.choices(TACTICS, weights=[self.mix[t] for t in TACTICS])[0] + + def update(self, outcomes: dict[str, tuple[int, int]]) -> dict[str, float]: + """`outcomes` maps a tactic to (scams run, scams that got through). Returns the new mix. + + Multiplicative weights on the success rate: a tactic that got through more + often gains share. A tactic tried fewer than `min_cases` times keeps its weight. + """ + cfg = self.cfg + if not cfg.adapt: + return dict(self.mix) + weights = {} + for t in TACTICS: + run, through = outcomes.get(t, (0, 0)) + gain = math.exp(cfg.learning_rate * through / run) if run >= cfg.min_cases else 1.0 + # Unproven tactics are scaled by the mean gain, so they neither win nor lose. + weights[t] = (self.mix[t], gain if run >= cfg.min_cases else None) + proven = [g for _, g in weights.values() if g is not None] + mean_gain = sum(proven) / len(proven) if proven else 1.0 + raw = {t: share * (mean_gain if g is None else g) for t, (share, g) in weights.items()} + total = sum(raw.values()) + free = 1.0 - cfg.floor * len(TACTICS) + self.mix = {t: cfg.floor + free * raw[t] / total for t in TACTICS} + return dict(self.mix) + + +class AdaptivePlanner(FraudPlanner): + """The default planner's state plus adaptive cells that start on `deploy_day`. + + Built from the running planner without drawing any random numbers, so the + world up to the deployment day is exactly the default one. + """ + + def __init__(self, base: FraudPlanner, adversary: Adversary, deploy_day: int) -> None: + self.sim = base.sim + self.mules = base.mules + self.victims = base.victims + self.cases = base.cases + self.cells = base.cells # shared: callbacks queued by `base` still find every cell + self.adversary = adversary + self.deploy_day = deploy_day + self.tactic: dict[int, str] = {} # case id -> tactic + self.reported: dict[int, float] = {} # case id -> when the victim first reported + self.first_adaptive = len(self.cells) + self.seasoned: dict[int, list[Mule]] = {} + self.spare: dict[int, list[Mule]] = {} + self._add_cells() + + @property + def round_start_day(self) -> int: + return self.deploy_day + self.adversary.cfg.warmup_days + + def is_adaptive(self, cell: Cell) -> bool: + return cell.id >= self.first_adaptive + + def _add_cells(self) -> None: + sim = self.sim + w, rng, acfg = sim.w, sim.rng, self.adversary.cfg + start = self.round_start_day + end = start + acfg.rounds * acfg.round_days - 1 + for i in range(acfg.n_cells): + a = BASE_TYPOLOGIES[i % 4] + b = BASE_TYPOLOGIES[(i // 4 + i + 1) % 4] + cell = Cell( + id=len(self.cells), + district=rng.randrange(len(DISTRICTS)), + start_day=start, + end_day=end, + typologies=(a, b) if a != b else (a,), + agents=[], + devices=[w.new_device() for _ in range(rng.randint(2, 4))], + recruited_share=rng.uniform(0.0, 0.35), + p_forward=rng.uniform(0.2, 0.5), + device_reuse=rng.uniform(0.3, 0.8), + heldout=False, + intensity=rng.uniform(0.7, 1.3), + ) + n_agents = rng.choices((0, 1, 2), weights=(0.35, 0.45, 0.20))[0] + free = [ + a + for a in w.agents_by_district[cell.district] + if not w.agent_colluding[a] and not w.agent_farming[a] + ] + for a in rng.sample(free, min(n_agents, len(free))): + w.agent_colluding[a] = True + cell.agents.append(a) + self.cells.append(cell) + self.seasoned[cell.id] = [] + self.spare[cell.id] = [] + + # ----------------------------------------------------------------- mules + + def _maintain_mules(self, cell: Cell, t0: int) -> None: + super()._maintain_mules(cell, t0) + if not self.is_adaptive(cell): + return + for pool, target, make in ( + (self.seasoned[cell.id], SEASONED_POOL, self._seasoned_mule), + (self.spare[cell.id], FAN_OUT_POOL, lambda c, t: self._create_mule(c, 1, t)), + ): + pool[:] = [m for m in pool if m.retire_ts > t0 and m.flagged_ts > t0] + ahead = sum(1 for m in pool if m.retire_ts > t0 + 6 * DAY) + for _ in range(target - ahead): + mule = make(cell, t0) + if mule is not None: + pool.append(mule) + + def _seasoned_mule(self, cell: Cell, t0: int) -> Mule | None: + """An old customer wallet with a normal history, taken over for the cell.""" + w, rng = self.sim.w, self.sim.rng + for _ in range(30): + c = rng.randrange(w.n_customers) + if ( + w.created_ts[c] < t0 - SEASONED_MIN_AGE_DAYS * DAY + and w.segment[c] in RECRUITABLE + and c not in self.mules + and c not in self.victims + and w.blocked_ts[c] == math.inf + ): + usable = t0 + rng.uniform(0, DAY) + mule = Mule(c, cell.id, 1, "seasoned", usable, usable + rng.uniform(10, 20) * DAY) + self.mules[c] = mule + return mule + return None + + # ----------------------------------------------------------------- scams + + def _new_case(self, cell: Cell, t0: int) -> None: + if not self.is_adaptive(cell): + super()._new_case(cell, t0) + return + sim = self.sim + rng, w, cap = sim.rng, sim.w, sim.cfg.txn_cap + tactic = self.adversary.draw() + typology = rng.choice(cell.typologies) + min_balance = {"impersonation": 1500, "wrong_send": 800, "account_takeover": 2000}.get( + typology, 0 + ) + need_app = typology == "account_takeover" + if tactic == "mimic_hours": + # Choose the victim first, then call at the hour they usually transact. + victim = self._pick_victim(t0, min_balance, need_app) + if victim is None: + return + hour = min(23.9, max(0.0, rng.gauss(float(w.peak_hour[victim]), 1.0))) + ts0 = t0 + int(hour * 3600) + else: + if typology == "account_takeover": + hour = rng.uniform(0, 6) if rng.random() < 0.5 else rng.uniform(0, 24) + else: + hour = min(22.0, max(9.0, rng.gauss(15, 3.5))) + ts0 = t0 + int(hour * 3600) + victim = None + + ready = self._ready(cell.l1, ts0) + if tactic == "seasoned_mule": + ready = self._ready(self.seasoned[cell.id], ts0) or ready + elif tactic == "fan_out": + ready = ready + self._ready(self.spare[cell.id], ts0) + if not ready: + if victim is not None: + self.victims.discard(victim) + return + if victim is None: + victim = self._pick_victim(ts0, min_balance, need_app) + if victim is None: + return + mule = rng.choice(ready) + + case = Case(len(self.cases), typology, cell.id, victim, mule.wallet, ts0) + self.cases.append(case) + self.tactic[case.id] = tactic + tag = {"case": case.id, "typology": typology, "cell": cell.id} + + sent_to: dict[int, int] = {} + + def receiver() -> int: + if tactic != "fan_out": + return mule.wallet + # The least-used ready mule takes the next payment. + least = min(ready, key=lambda m: (m.n_victims + sent_to.get(m.wallet, 0), rng.random())) + sent_to[least.wallet] = sent_to.get(least.wallet, 0) + 1 + return least.wallet + + def amounts( + amount: float | None, frac: float | None + ) -> list[tuple[float | None, float | None]]: + """One transfer, or with split_amounts several below a cap.""" + if tactic != "split_amounts": + return [(amount, frac)] + limit = rng.choice(SPLIT_CAPS) + total = amount if amount is not None else w.balance[victim] * (frac or 0.0) + k = max(2, min(6, math.ceil(total / limit))) + part = max(math.floor(total / k / 10) * 10, 50.0) + return [(float(part), None)] * k + + def victim_send(t: float, amount: float | None = None, frac: float | None = None, + **kw) -> float: # fmt: skip + for a, f in amounts(amount, frac): + sim.push( + t, + Intent(SEND, victim, receiver(), a or 0.0, frac=f, role="victim_transfer", + fraud=True, **tag, **kw), + ) # fmt: skip + t += rng.uniform(300, 1800) + return t + + def bait(t: float, amount: float) -> None: + sim.push(t, Intent(SEND, mule.wallet, victim, amount, role="bait", **tag)) + + if typology == "impersonation": + t = ts0 + for _ in range(rng.choices((1, 2, 3), weights=(0.55, 0.30, 0.15))[0]): + t = victim_send(t, frac=rng.uniform(0.5, 0.97)) + t += rng.uniform(60, 480) + + elif typology == "wrong_send": + if rng.random() < 0.15: + bait(ts0, float(rng.choice((20, 50, 100)))) + amount = rng.choices( + (1000, 1500, 2000, 3000, 5000, 8000, 10000, 15000), + weights=(18, 12, 20, 16, 16, 8, 7, 3), + )[0] + victim_send(ts0 + rng.uniform(300, 3600), amount=float(amount)) + + elif typology == "lottery_fee": + t, amount = ts0, float(rng.choice((500, 1000, 1500, 2000))) + for _ in range(rng.randint(2, 5)): + t = victim_send(t, amount=min(amount, cap), topup=True) + if rng.random() < 0.25: + break + amount = round(amount * rng.uniform(1.4, 2.4), -2) + t += rng.uniform(1200, 20 * 3600) + + else: # account_takeover + device = rng.choice(cell.devices) if rng.random() < 0.5 else w.new_device() + direct = bool(cell.agents) and rng.random() < 0.3 + t = ts0 + for _ in range(rng.randint(1, 4)): + for a, f in amounts(None, rng.uniform(0.5, 1.0)): + kw = dict(frac=f, device=device, district=cell.district, + role="ato_transfer", fraud=True, **tag) # fmt: skip + if direct: + sim.push( + t, Intent(CASH_OUT, victim, rng.choice(cell.agents), a or 0.0, **kw) + ) + else: + sim.push(t, Intent(SEND, victim, receiver(), a or 0.0, **kw)) + t += ( + rng.uniform(40, 300) + if tactic != "split_amounts" + else rng.uniform(300, 1800) + ) + + # -------------------------------------------------------------- reaction + + def on_case_transfer(self, ts: int, it: Intent, receiver: int | None, amount: float) -> None: + cell = self.cells[it.cell] + if not self.is_adaptive(cell): + super().on_case_transfer(ts, it, receiver, amount) + return + sim = self.sim + rng = sim.rng + tactic = self.tactic.get(it.case, "none") + if it.role in VICTIM_ROLES: + case = self.cases[it.case] + case.loss += amount + case.n_txn += 1 + if receiver is None or it.role == "bait": + return + mule = self.mules.get(receiver) + if mule is None: + return + mule.first_fraud_ts = min(mule.first_fraud_ts, ts) + mule.last_fraud_ts = max(mule.last_fraud_ts, ts) + tag = {"case": it.case, "typology": it.typology, "cell": it.cell, "fraud": True} + + if it.role in VICTIM_ROLES: + mule.n_victims += 1 + if rng.random() < sim.cfg.report_rate: + delay = rng.lognormvariate(math.log(1.5 * DAY), 0.8) + rng.uniform(3600, 12 * 3600) + first = self.reported.get(it.case, math.inf) + self.reported[it.case] = min(first, ts + delay) + sim.push(ts + delay, lambda t, mu=mule: self._flag(mu, t, "victim_report")) + + out = math.floor(amount * rng.uniform(0.92, 1.0) / 10) * 10 + if out < 50: + return + if tactic == "delayed_cashout": + dwell = rng.uniform(*DELAY_HOURS) * 3600 + elif rng.random() < 0.1: + dwell = rng.uniform(3600, 8 * 3600) + else: + dwell = rng.uniform(60, 1500) + + l2 = self._ready(cell.l2, ts) if mule.level == 1 else [] + if l2 and rng.random() < cell.p_forward: + nxt = rng.choice(l2).wallet + sim.push( + ts + dwell, Intent(SEND, receiver, nxt, float(out), role="mule_forward", **tag) + ) + return + + w = sim.w + if cell.agents and rng.random() < 0.8: + agent = rng.choice(cell.agents) + else: + agent = rng.choice(w.agents_by_district[w.district[receiver]]) + chunks = [out] + if out > 12_000 and rng.random() < 0.5: + first_part = math.floor(out * rng.uniform(0.35, 0.65) / 100) * 100 + chunks = [first_part, out - first_part] + t = ts + dwell + for chunk in chunks: + sim.push(t, Intent(CASH_OUT, receiver, agent, float(chunk), role="mule_cashout", **tag)) + t += rng.uniform(60, 400) + + +def deploy(sim, adversary: Adversary, deploy_day: int) -> AdaptivePlanner: + """Swap the running simulation's fraud planner for an adaptive one.""" + planner = AdaptivePlanner(sim.fraud, adversary, deploy_day) + sim.fraud = planner + return planner diff --git a/backend/src/fraudlens/simulator/config.py b/backend/src/fraudlens/simulator/config.py index 32c5189..6149c71 100644 --- a/backend/src/fraudlens/simulator/config.py +++ b/backend/src/fraudlens/simulator/config.py @@ -1,10 +1,17 @@ """Simulation parameters. Every number here is a documented synthetic assumption -(see docs/DATA_ASSUMPTIONS.md), not a measurement of any real MFS.""" +(see docs/DATA_ASSUMPTIONS.md), not a measurement of any real MFS. -from dataclasses import dataclass +Two profiles exist. "default" is the world every published number comes from. +"calibrated" replaces the parameters that a public Bangladesh source pins down +(DATA_ASSUMPTIONS.md §11, sources in docs/BANGLADESH_CONTEXT.md) and leaves +everything else as it is. +""" + +from dataclasses import dataclass, replace from datetime import datetime DAY = 86_400 +PROFILES = ("default", "calibrated") @dataclass(frozen=True) @@ -35,6 +42,16 @@ class SimConfig: max_mule_wallets: int = 3_000 farming_agent_share: float = 0.015 + profile: str = "default" + # Median (BDT, before the segment multiplier) of spontaneous send-money and + # cash-out amounts. The calibrated profile moves them toward Bangladesh Bank's + # published average ticket sizes. + send_median: float = 1_500.0 + cash_out_median: float = 3_000.0 + # Share of account-takeover cases kept; the rest are run as impersonation + # instead. 1.0 leaves the typology mix (and the random stream) untouched. + ato_keep_share: float = 1.0 + def split_of(self, day: int) -> str: if day < self.train_end_day: return "train" @@ -58,3 +75,27 @@ def small(cls, seed: int = 7) -> "SimConfig": n_heldout_cells=1, max_mule_wallets=600, ) + + def calibrated(self) -> "SimConfig": + """The same world with every parameter that has a public Bangladesh source + set from it. Values and sources: DATA_ASSUMPTIONS.md §11.""" + return replace( + self, + profile="calibrated", + # Smallest Bangladesh Bank daily limit from 27 March 2025 (cash-out, Tk 30,000). + txn_cap=30_000.0, + # Average P2P ticket Tk 3,552 and cash-out Tk 2,132 (Bangladesh Bank, Oct 2025). + # The send median puts the simulated send mean at about Tk 3,520. Most cash-outs + # are balance-driven withdrawals, so the cash-out mean barely moves (§11). + send_median=2_850.0, + cash_out_median=1_400.0, + # 41.2% of personal account holders filed a complaint (TIB 2025). + report_rate=0.412, + # Hacking is 12.3% of fraud reports (TIB 2025), half the default ATO share. + ato_keep_share=0.5, + ) + + def with_profile(self, profile: str) -> "SimConfig": + if profile not in PROFILES: + raise ValueError(f"unknown simulator profile {profile!r}; choose from {PROFILES}") + return self.calibrated() if profile == "calibrated" else self diff --git a/backend/src/fraudlens/simulator/engine.py b/backend/src/fraudlens/simulator/engine.py index 1eea0f8..c94079a 100644 --- a/backend/src/fraudlens/simulator/engine.py +++ b/backend/src/fraudlens/simulator/engine.py @@ -125,7 +125,12 @@ def push(self, ts: float, item: object) -> None: heapq.heappush(self._heap, (int(ts), self._seq, item)) def run(self) -> None: - for day in range(self.cfg.days): + self.run_days(0, self.cfg.days) + + def run_days(self, start: int, stop: int) -> None: + """Simulate days [start, stop). Called in order, it continues where it stopped, + which is how the adversary experiment steps the world a round at a time.""" + for day in range(start, stop): self._schedule_day(day) self.fraud.schedule_day(day) end = (day + 1) * DAY @@ -187,7 +192,7 @@ def _schedule_day(self, day: int) -> None: continue times = self._times(idx, t0).tolist() if typ == SEND: - amounts = self._amounts(idx, 1500, 0.9, 50) + amounts = self._amounts(idx, cfg.send_median, 0.9, 50) elif typ == PAYMENT: amounts = self._amounts(idx, 450, 0.8, 20) elif typ == RECHARGE: @@ -195,7 +200,7 @@ def _schedule_day(self, day: int) -> None: elif typ == BILL_PAY: amounts = self._amounts(idx, 900, 0.6, 100) elif typ == CASH_OUT: - amounts = self._amounts(idx, 3000, 0.7, 500) + amounts = self._amounts(idx, cfg.cash_out_median, 0.7, 500) else: amounts = self._amounts(idx, 2500, 0.7, 500) topup = typ in (SEND, PAYMENT, RECHARGE, BILL_PAY) diff --git a/backend/src/fraudlens/simulator/fraud.py b/backend/src/fraudlens/simulator/fraud.py index 75f55de..a946f37 100644 --- a/backend/src/fraudlens/simulator/fraud.py +++ b/backend/src/fraudlens/simulator/fraud.py @@ -228,6 +228,12 @@ def _new_case(self, cell: Cell, t0: int) -> None: sim = self.sim rng = sim.rng typology = rng.choice(cell.typologies) + if ( + typology == "account_takeover" + and sim.cfg.ato_keep_share < 1.0 + and rng.random() >= sim.cfg.ato_keep_share + ): + typology = "impersonation" # calibrated profile: fewer takeovers, more deception if typology == "account_takeover": hour = rng.uniform(0, 6) if rng.random() < 0.5 else rng.uniform(0, 24) else: diff --git a/backend/src/fraudlens/simulator/generate.py b/backend/src/fraudlens/simulator/generate.py index 8126892..74fb8c3 100644 --- a/backend/src/fraudlens/simulator/generate.py +++ b/backend/src/fraudlens/simulator/generate.py @@ -17,7 +17,7 @@ import pandas as pd from ..config import settings -from .config import DAY, SimConfig +from .config import DAY, PROFILES, SimConfig from .engine import ADD_MONEY, CASH_IN, K_WALLET, KIND_NAMES, TYPE_NAMES, Simulation from .world import DISTRICTS, MERCHANT_CATEGORIES, SEGMENTS @@ -39,21 +39,23 @@ def _when(cfg: SimConfig, seconds) -> pd.Series: return pd.Timestamp(cfg.start) + pd.to_timedelta(s.round(), unit="s") -def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: - cfg, w, c = sim.cfg, sim.w, sim.cols +def transaction_table(sim: Simulation, start: int = 0) -> pd.DataFrame: + """The transactions emitted so far, from the `start`-th one on (txn_id = position).""" + cfg = sim.cfg + c = {k: v[start:] for k, v in sim.cols.items()} ts = np.asarray(c["ts"], dtype=np.int64) day = ts // DAY - typ = np.asarray(c["type"]) + typ = np.asarray(c["type"], dtype=np.int64) app = np.asarray(c["app"]) - device = np.asarray(c["device"]) + device = np.asarray(c["device"], dtype=np.int64) channel = np.where(app == 1, "app", "ussd").astype(object) channel[typ == CASH_IN] = "agent" channel[typ == ADD_MONEY] = "bank" - txns = pd.DataFrame( + return pd.DataFrame( { - "txn_id": np.arange(len(ts), dtype=np.int64), + "txn_id": np.arange(start, start + len(ts), dtype=np.int64), "ts": _when(cfg, ts), "day": day, "split": np.select( @@ -77,6 +79,11 @@ def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: } ) + +def build_tables(sim: Simulation) -> dict[str, pd.DataFrame]: + cfg, w = sim.cfg, sim.w + txns = transaction_table(sim) + nw = w.n_wallets mules = sim.fraud.mules used = {wid: m for wid, m in mules.items() if m.n_victims > 0 or m.first_fraud_ts < math.inf} @@ -210,9 +217,19 @@ def main() -> None: parser.add_argument("--small", action="store_true", help="test-sized world") parser.add_argument("--seed", type=int, default=7) parser.add_argument("--out", type=Path, default=None) + parser.add_argument( + "--profile", + choices=PROFILES, + default=settings.sim_profile, + help="calibrated: Bangladesh-sourced parameters (env FRAUDLENS_SIM_PROFILE)", + ) args = parser.parse_args() cfg = SimConfig.small(args.seed) if args.small else SimConfig(seed=args.seed) - out = args.out or settings.data_dir / ("small" if args.small else "full") + cfg = cfg.with_profile(args.profile) + name = "small" if args.small else "full" + if cfg.profile != "default": + name = f"{name}_{cfg.profile}" # never overwrite the published dataset + out = args.out or settings.data_dir / name meta = generate(cfg, out) print(json.dumps(meta["profile"], indent=2, default=str)) print(f"wrote {out} in {meta['seconds']}s") diff --git a/backend/src/fraudlens/validation/__init__.py b/backend/src/fraudlens/validation/__init__.py new file mode 100644 index 0000000..6fd44b9 --- /dev/null +++ b/backend/src/fraudlens/validation/__init__.py @@ -0,0 +1 @@ +"""Tools for validating FraudLens with real people: survey and interview analysis.""" diff --git a/backend/src/fraudlens/validation/survey_analysis.py b/backend/src/fraudlens/validation/survey_analysis.py new file mode 100644 index 0000000..7e3b683 --- /dev/null +++ b/backend/src/fraudlens/validation/survey_analysis.py @@ -0,0 +1,140 @@ +"""Summarise the customer survey (docs/validation/customer_survey.md). + +uv run python -m fraudlens.validation.survey_analysis responses.csv +uv run python -m fraudlens.validation.survey_analysis responses.csv --by area + +One row per respondent, one column per question ID in the survey. Rows without +consent are dropped before anything is counted. Free text (q14) is never read. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pandas as pd + +REQUIRED = ("respondent_id", "consent") +CATEGORICAL = { + "area": ("urban", "rural"), + "q01_contacted": ("yes", "no"), + "q02_lost_money": ("yes", "no"), + "q03_scam_type": ( + "impersonation", + "lottery_fee", + "wrong_send", + "investment", + "account_takeover", + "other", + ), + "q05_reported_to": ("nobody", "provider", "police", "cipc", "several"), + "q06_recovered": ("none", "partial", "full"), + "q12_trust": ("more", "same", "less"), + "q13_language": ("bangla", "english", "both"), +} +LIKERT = { + "q07_warn_would_stop": "warn", + "q08_step_up_ok": "step_up", + "q09_hold_ok": "hold", +} +NUMERIC = ("q04_loss_bdt", "q10_max_hold_minutes", "q11_false_alarms_per_year") + + +class SurveyError(ValueError): + pass + + +def load(path: Path) -> pd.DataFrame: + df = pd.read_csv(path, dtype=str, keep_default_na=False) + missing = [c for c in REQUIRED if c not in df.columns] + if missing: + raise SurveyError(f"missing columns: {', '.join(missing)}") + if df["respondent_id"].duplicated().any(): + raise SurveyError("respondent_id is not unique") + for col in CATEGORICAL: + if col in df: + df[col] = df[col].str.strip().str.lower() + return df + + +def _share(s: pd.Series, levels: tuple[str, ...]) -> dict: + s = s[s != ""] + unknown = sorted(set(s) - set(levels)) + if unknown: + raise SurveyError(f"unexpected answers {unknown} in {s.name}") + if s.empty: + return {"n": 0} + counts = s.value_counts() + return {"n": int(len(s)), **{k: round(int(counts.get(k, 0)) / len(s), 3) for k in levels}} + + +def _numbers(s: pd.Series) -> dict: + x = pd.to_numeric(s.replace("", None), errors="coerce").dropna() + if (x < 0).any(): + raise SurveyError(f"negative values in {s.name}") + if x.empty: + return {"n": 0} + return { + "n": int(len(x)), + "median": float(x.median()), + "mean": round(float(x.mean()), 1), + "p90": float(x.quantile(0.9)), + } + + +def _likert(s: pd.Series) -> dict: + x = pd.to_numeric(s.replace("", None), errors="coerce").dropna() + if not x.between(1, 5).all(): + raise SurveyError(f"{s.name} must be 1-5") + if x.empty: + return {"n": 0} + return { + "n": int(len(x)), + "mean": round(float(x.mean()), 2), + "agree": round(float((x >= 4).mean()), 3), # 4 or 5 + "disagree": round(float((x <= 2).mean()), 3), + } + + +def summarise(df: pd.DataFrame) -> dict: + consented = df[df["consent"].str.strip().str.lower() == "yes"] + out: dict = {"responses": len(df), "consented": len(consented)} + for col, levels in CATEGORICAL.items(): + if col in consented: + out[col] = _share(consented[col], levels) + # Scam type, loss, reporting and recovery describe victims only. + lost = consented.get("q02_lost_money") + victims = consented[lost == "yes"] if lost is not None else consented.iloc[0:0] + for col in ("q03_scam_type", "q05_reported_to", "q06_recovered"): + if col in victims: + out[col] = _share(victims[col], CATEGORICAL[col]) + for col in NUMERIC: + if col in consented: + src = victims if col == "q04_loss_bdt" else consented + out[col] = _numbers(src[col]) + out["screens"] = { + name: _likert(consented[col]) for col, name in LIKERT.items() if col in consented + } + return out + + +def summarise_by(df: pd.DataFrame, column: str) -> dict: + if column not in df: + raise SurveyError(f"no column {column}") + groups = {str(k): summarise(g) for k, g in df.groupby(column, sort=True)} + return {"all": summarise(df), "by": column, "groups": groups} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarise FraudLens customer survey responses") + parser.add_argument("csv", type=Path) + parser.add_argument("--by", default=None, help="also split by this column, e.g. area") + args = parser.parse_args() + df = load(args.csv) + result = summarise_by(df, args.by) if args.by else summarise(df) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/fixtures/survey_fake.csv b/backend/tests/fixtures/survey_fake.csv new file mode 100644 index 0000000..d0575f4 --- /dev/null +++ b/backend/tests/fixtures/survey_fake.csv @@ -0,0 +1,6 @@ +respondent_id,consent,area,q01_contacted,q02_lost_money,q03_scam_type,q04_loss_bdt,q05_reported_to,q06_recovered,q07_warn_would_stop,q08_step_up_ok,q09_hold_ok,q10_max_hold_minutes,q11_false_alarms_per_year,q12_trust,q13_language,q14_comments +FAKE-001,yes,urban,yes,yes,impersonation,5000,provider,none,5,4,3,30,2,more,bangla,FAKE ROW - not a real person +FAKE-002,yes,urban,yes,no,,,,,4,5,2,10,1,same,both,FAKE ROW - not a real person +FAKE-003,yes,rural,yes,yes,lottery_fee,1000,nobody,none,2,3,1,0,0,less,bangla,FAKE ROW - not a real person +FAKE-004,yes,rural,no,no,,,,,5,5,4,60,3,more,bangla,FAKE ROW - not a real person +FAKE-005,no,urban,yes,yes,wrong_send,99999,police,full,1,1,1,0,0,less,english,FAKE ROW - declined consent so must be ignored diff --git a/backend/tests/test_adversary.py b/backend/tests/test_adversary.py new file mode 100644 index 0000000..5a159ff --- /dev/null +++ b/backend/tests/test_adversary.py @@ -0,0 +1,123 @@ +import math + +import pytest + +from fraudlens.models import adversary as experiment +from fraudlens.simulator.adversary import ( + DELAY_HOURS, + SEASONED_MIN_AGE_DAYS, + TACTICS, + Adversary, + AdversaryConfig, + deploy, +) +from fraudlens.simulator.config import SimConfig +from fraudlens.simulator.engine import Simulation +from fraudlens.simulator.generate import build_tables, transaction_table + +DAY = 86_400 + + +def test_stepping_the_world_by_days_matches_one_run(small_tables, small_cfg): + sim = Simulation(small_cfg) + sim.run_days(0, 20) + sim.run_days(20, small_cfg.days) + txns = build_tables(sim)["transactions"] + expected = small_tables["transactions"] + assert len(txns) == len(expected) + assert txns["amount"].sum() == pytest.approx(expected["amount"].sum()) + + +def test_mix_moves_toward_what_got_through(): + adv = Adversary(AdversaryConfig()) + start = dict(adv.mix) + outcomes = {t: (20, 0) for t in TACTICS} + outcomes["fan_out"] = (20, 15) + mix = adv.update(outcomes) + assert mix["fan_out"] > start["fan_out"] + assert mix["none"] < start["none"] + assert sum(mix.values()) == pytest.approx(1.0) + assert min(mix.values()) >= AdversaryConfig().floor + + +def test_too_few_cases_say_nothing_and_control_never_adapts(): + adv = Adversary(AdversaryConfig()) + before = dict(adv.mix) + after = adv.update({t: (2, 2) for t in TACTICS}) # below min_cases everywhere + # Only the floor moves it: no tactic gains on another. + assert max(after, key=after.get) == max(before, key=before.get) + assert len({round(after[t], 9) for t in TACTICS if t != "none"}) == 1 + control = Adversary(AdversaryConfig(adapt=False)) + assert control.update({"fan_out": (50, 50)}) == control.mix + + +@pytest.fixture(scope="module") +def adaptive_world(): + """The small world's last 20 days with adaptive cells deployed.""" + cfg = SimConfig.small() + sim = Simulation(cfg) + deploy_day = cfg.days - 20 + sim.run_days(0, deploy_day) + start = len(sim.cols["amount"]) + planner = deploy(sim, Adversary(AdversaryConfig(n_cells=4, warmup_days=4)), deploy_day) + sim.run_days(deploy_day, cfg.days) + return sim, planner, transaction_table(sim, start) + + +def _rows(txns, planner, tactic, role): + cases = [c for c, t in planner.tactic.items() if t == tactic] + return txns[txns["case_id"].isin(cases) & (txns["fraud_role"] == role)] + + +def test_adaptive_cells_try_every_tactic(adaptive_world): + _, planner, txns = adaptive_world + assert set(planner.tactic.values()) == set(TACTICS) + assert txns["case_id"].isin(list(planner.tactic)).any() + + +def test_split_payments_are_smaller_and_more_numerous(adaptive_world): + _, planner, txns = adaptive_world + split = _rows(txns, planner, "split_amounts", "victim_transfer") + plain = _rows(txns, planner, "none", "victim_transfer") + assert len(split) and len(plain) + assert split["amount"].median() < plain["amount"].median() + assert split.groupby("case_id").size().mean() > plain.groupby("case_id").size().mean() + + +def test_delayed_cash_out_waits_hours(adaptive_world): + _, planner, txns = adaptive_world + first_in = _rows(txns, planner, "delayed_cashout", "victim_transfer") + first_out = _rows(txns, planner, "delayed_cashout", "mule_cashout") + first_in = first_in.groupby("case_id")["ts"].min() + first_out = first_out.groupby("case_id")["ts"].min() + both = first_in.index.intersection(first_out.index) + assert len(both) + waited = (first_out[both] - first_in[both]).dt.total_seconds() + assert (waited >= DELAY_HOURS[0] * 3600).all() + + +def test_seasoned_mules_are_old_wallets(adaptive_world): + sim, planner, _ = adaptive_world + seasoned = [m for pool in planner.seasoned.values() for m in pool] + assert seasoned + for m in seasoned: + assert m.kind == "seasoned" + assert m.usable_ts - float(sim.w.created_ts[m.wallet]) >= SEASONED_MIN_AGE_DAYS * DAY + + +def test_experiment_runs_end_to_end(trained): + data_dir, models_root, _ = trained + acfg = AdversaryConfig(rounds=2, round_days=3, n_cells=3, warmup_days=5) + report = experiment.run(data_dir, models_root, None, acfg, log=lambda *_: None) + assert set(report["arms"]) == set(experiment.ARMS) + for arm in experiment.ARMS: + rounds = report["arms"][arm]["rounds"] + assert len(rounds) == acfg.rounds + for r in rounds: + for key in ("case_recall", "loss_txn_recall"): + assert r[key] is None or 0 <= r[key] <= 1 + control = report["arms"]["control"]["rounds"] + assert control[0]["mix"] == control[-1]["mix_after"] + assert report["arms"]["frozen"]["last_model"] == report["served_model"] + assert report["arms"]["retrain"]["last_model"] == "retrain-r1" + assert not math.isnan(report["seconds"]) diff --git a/backend/tests/test_business.py b/backend/tests/test_business.py new file mode 100644 index 0000000..a69ebf7 --- /dev/null +++ b/backend/tests/test_business.py @@ -0,0 +1,170 @@ +"""The business case: the threshold sweep priced in taka.""" + +import json + +import pytest +from pydantic import ValidationError + +from fraudlens.config import settings +from fraudlens.decision import business +from fraudlens.decision.business import ASSUMPTIONS, Assumptions, business_case, sensitivity +from fraudlens.models import registry + + +def _row(threshold, alert_rate, precision, recall, recall_exit): + return { + "threshold": threshold, + "alert_rate": alert_rate, + "precision": precision, + "taka_recall": recall, + "taka_recall_with_exit_holds": recall_exit, + } + + +@pytest.fixture +def report(): + """1,000 payments, ৳10,000 of scam money; tiers at 0.1 / 0.5 / 0.9.""" + hold = _row(0.9, 0.004, 1.0, 0.5, 0.6) + step_up = _row(0.5, 0.006, 0.8, 0.7, 0.8) + warn = {**_row(0.1, 0.010, 0.6, 0.8, 0.9), "taka_at_risk": 10_000} + return { + "model_version": "v9", + "policy_version": "v1", + "rows": 1_000, + "days": 10, + "thresholds": {"warn": 0.1, "step_up": 0.5, "hold": 0.9}, + "at_thresholds": {"warn": warn, "step_up": step_up, "hold": hold}, + "impact": [_row(0.95, 0.002, 1.0, 0.3, 0.35), warn, _row(0.01, 0.05, 0.1, 0.9, 0.95)], + } + + +def _plain(**changes) -> Assumptions: + """Round numbers so the arithmetic can be checked by hand.""" + values = dict( + monthly_payments=1_000_000, + avg_payment_taka=1_000, + scam_loss_bps=100, # ৳10 a payment, the same as the report: scale 1 + stop_warn=0.2, + stop_step_up=0.5, + stop_hold=1.0, + abandon_rate=0.1, + abandon_cost_taka=10, + contacts_warn=0.0, + contacts_step_up=0.0, + contacts_hold=1.0, + contact_cost_taka=5, + review_minutes=6, + analyst_monthly_cost_taka=50_000, + analyst_hours_per_month=100, + min_analysts=0, + platform_monthly_cost_taka=100_000, + reimbursement_share=0.5, + reputation_per_taka=0.0, + ) + return Assumptions(**(values | changes)) + + +def test_the_policy_point_adds_up_by_hand(report): + case = business_case(report, _plain()) + p = case["policy"] + assert case["synthetic"]["scale"] == 1.0 and case["scam_loss_at_risk"] == 10_000_000 + # Bands of recall: warn 0.8-0.7, step-up 0.7-0.5, hold 0.5; exit holds 0.6-0.5. + assert p["prevented"] == { + "warn": 200_000, + "step_up": 1_000_000, + "hold": 5_000_000, + "mule_cash_out_held": 1_000_000, + } + assert p["prevented_total"] == 7_200_000 and p["missed"] == 2_800_000 + # Honest: warn band 0.004-0.0012, step-up 0.0012-0, hold 0 → 4,000 a month. + assert p["honest_interrupted"] == 4_000 and p["honest_per_10k"] == 40 + assert p["interrupted"] == {"warn": 4_000, "step_up": 2_000, "hold": 4_000} + assert p["friction"] == {"abandoned_payments": 4_000, "support_contacts": 20_000} + # 4,000 holds x 6 minutes = 400 hours = 4 analysts. + assert (p["review_hours"], p["analysts"], p["analyst_cost"]) == (400, 4, 200_000) + assert p["operating_cost"] == 324_000 + assert p["net_benefit"] == 7_200_000 - 324_000 + assert p["provider_net_benefit"] == 3_600_000 - 324_000 + assert p["prevented_per_taka_cost"] == round(7_200_000 / 324_000, 2) + + +def test_tiers_collapse_when_the_slider_passes_them(report): + case = business_case(report, _plain()) + strict = case["points"][0] # above the hold threshold: everything alerted is held + assert strict["prevented"]["warn"] == strict["prevented"]["step_up"] == 0 + assert strict["interrupted"]["hold"] == 2_000 == strict["interrupted_total"] + assert case["points"][1]["prevented_total"] == case["policy"]["prevented_total"] + + +def test_a_rarer_scam_rescales_true_alerts_but_not_false_ones(report): + dense, rare = (business_case(report, _plain(scam_loss_bps=b))["policy"] for b in (100, 10)) + assert rare["prevented_total"] == dense["prevented_total"] / 10 + assert rare["honest_interrupted"] == dense["honest_interrupted"] + assert rare["interrupted"]["hold"] == 400 # 4,000 true holds, a tenth as many + assert rare["analysts"] == 1 + + +def test_coverage_floor_and_friction_move_the_net_as_expected(report): + base = business_case(report, _plain())["policy"] + floored = business_case(report, _plain(min_analysts=10))["policy"] + assert floored["analysts"] == 10 and floored["net_benefit"] == base["net_benefit"] - 300_000 + sticky = business_case(report, _plain(abandon_rate=0.5))["policy"] + assert sticky["friction"]["abandoned_payments"] == 20_000 + + +def test_sensitivity_is_sorted_and_reimbursement_moves_only_the_provider(report): + table = sensitivity(report, _plain()) + swings = [row["swing"] for row in table["rows"]] + assert swings == sorted(swings, reverse=True) + assert {row["key"] for row in table["rows"]} == set(ASSUMPTIONS) + reimburse = next(r for r in table["rows"] if r["key"] == "reimbursement_share") + assert reimburse["swing"] == 0 and reimburse["provider_swing"] > 0 + + +def test_break_even_is_where_the_net_crosses_zero(report): + a = _plain() + bps = business.break_even_bps(report, a) + assert bps is not None and 0 < bps < 100 + at = business_case(report, a.model_copy(update={"scam_loss_bps": bps * 1.01}))["policy"] + below = business_case(report, a.model_copy(update={"scam_loss_bps": bps * 0.99}))["policy"] + assert at["net_benefit"] >= 0 > below["net_benefit"] + + +def test_assumptions_are_bounded_and_strict(): + with pytest.raises(ValidationError): + Assumptions(abandon_rate=1.5) + with pytest.raises(ValidationError): + Assumptions(monthly_payments=-1) + with pytest.raises(ValidationError): + Assumptions(wage=1) # unknown fields are refused, not ignored + for key, spec in ASSUMPTIONS.items(): + assert spec.low <= getattr(Assumptions(), key) <= spec.high, key + assert spec.source, key + assert set(ASSUMPTIONS) == set(Assumptions.model_fields) + + +def test_a_report_without_a_sweep_is_refused(report): + with pytest.raises(ValueError): + business_case({**report, "impact": []}) + + +def test_on_the_served_model_the_policy_pays_for_itself(): + version = registry.current_version(settings.models_dir) + path = settings.models_dir / str(version) / "insights.json" + if not version or not path.exists(): + pytest.skip("no evaluated model in artifacts/") + report = json.loads(path.read_text()) + case = business_case(report) + p = case["policy"] + assert len(case["points"]) == len(report["impact"]) + assert 0 < p["prevented_total"] <= case["scam_loss_at_risk"] + assert p["net_benefit"] > 0 and p["prevented_per_taka_cost"] > 1 + assert p["analysts"] >= Assumptions().min_analysts + # Honest payments interrupted per 10,000 is the back-test's false-alert rate. + warn = report["at_thresholds"]["warn"] + false_rate = warn["alert_rate"] * (1 - warn["precision"]) + assert p["honest_per_10k"] == pytest.approx(false_rate * 1e4, abs=0.01) + prevented = [point["prevented_total"] for point in case["points"]] + assert prevented == sorted(prevented) # looser thresholds never stop less + # The scam rate is the assumption that matters most, and the least known. + assert case["sensitivity"]["rows"][0]["key"] == "scam_loss_bps" diff --git a/backend/tests/test_consortium.py b/backend/tests/test_consortium.py new file mode 100644 index 0000000..1155d5b --- /dev/null +++ b/backend/tests/test_consortium.py @@ -0,0 +1,278 @@ +"""The consortium protocol: privacy, integrity, reversibility and the audit trail.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from fraudlens.consortium import crypto +from fraudlens.consortium.hub import DISPUTE_SLA, Hub, Member +from fraudlens.consortium.protocol import ( + Bundle, + ProtocolError, + canonical, + group_element, + normalise, + signal, +) + +T0 = datetime(2026, 4, 1, tzinfo=UTC) +MULE = "01712345678" +MULE_PHONE = "8801712345678" +CUSTOMER = "01898765432" + + +@pytest.fixture(scope="module") +def keys(): + # Key generation is cheap but signing keys are reused to keep the module quick. + return {name: crypto.SigningKey.generate() for name in ("bkash", "nagad", "rocket")} + + +@pytest.fixture +def world(keys): + hub = Hub("2026Q2", T0) + members = {} + for name, key in keys.items(): + members[name] = Member(name, hub, key) + hub.register(name, name.title(), key.public, T0) + return hub, members + + +def test_group_prime_is_the_rfc_3526_group(): + assert crypto.P.bit_length() == 2048 + assert pow(crypto.G, crypto.Q, crypto.P) == 1 # g generates the order-q subgroup + + +def test_identifiers_are_normalised_before_hashing(): + assert normalise("msisdn", "+880 1712-345678") == MULE_PHONE + assert normalise("msisdn", MULE) == normalise("msisdn", MULE_PHONE) + with pytest.raises(ProtocolError): + normalise("msisdn", "0123") + with pytest.raises(ProtocolError): + normalise("email", "a@b.c") + + +def test_blinded_oprf_equals_the_keyed_function_and_hides_the_input(): + key = crypto.OprfKey.generate("e1") + element = group_element("msisdn", MULE) + blinded, r = crypto.blind(element) + assert blinded != element # the hub sees a random group element, not the number's hash + expected = pow(element, key.secret, crypto.P) + assert crypto.unblind(key.evaluate(blinded), r, key.public) == expected + # Blinding is fresh every time: two requests for one number are unlinkable. + assert crypto.blind(element)[0] != crypto.blind(element)[0] + + +def test_tokens_depend_on_the_hub_key(world): + hub, m = world + a = m["bkash"].tokenize([("msisdn", MULE)], T0)[0] + b = m["nagad"].tokenize([("msisdn", "+8801712345678")], T0)[0] + assert a == b # two providers, one SIM, one token + other = Member("x", Hub("2026Q2", T0), crypto.SigningKey.generate()) + other.hub.register("x", "X", other.key.public, T0) + assert other.tokenize([("msisdn", MULE)], T0)[0] != a # no key, no matching token + assert MULE_PHONE not in a and len(a) == 32 + + +def test_quota_stops_enumeration(world): + hub, m = world + hub.members["rocket"].quota_per_day = 3 + m["rocket"].tokenize([("msisdn", f"0171000000{i}") for i in range(3)], T0) + with pytest.raises(ProtocolError, match="quota"): + m["rocket"].tokenize([("msisdn", "01710000009")], T0) + assert hub.audit.entries[-1]["event"] == "oprf.refused" + + +def test_signatures_reject_forgery_and_tampering(keys): + msg = canonical({"a": 1}) + sig = keys["bkash"].sign(msg) + assert crypto.verify(keys["bkash"].public, msg, sig) + assert not crypto.verify(keys["nagad"].public, msg, sig) + assert not crypto.verify(keys["bkash"].public, canonical({"a": 2}), sig) + assert not crypto.verify(keys["bkash"].public, msg, "zz.zz") + + +def test_bloom_filter_has_no_false_negatives(): + bloom = crypto.BloomFilter.sized(500, 1e-4) + for i in range(500): + bloom.add(f"t{i}") + assert all(f"t{i}" in bloom for i in range(500)) + false = sum(f"u{i}" in bloom for i in range(20_000)) + assert false <= 20 # expected about 2 + again = crypto.BloomFilter.from_dict(bloom.to_dict()) + assert "t7" in again and again.n == 500 + + +def test_share_import_lookup_end_to_end(world): + hub, m = world + bkash, nagad = m["bkash"], m["nagad"] + bkash.list_wallet([("msisdn", MULE), ("device", "handset-77")], "confirmed", 0.9, T0, + typology="impersonation") # fmt: skip + bkash.list_wallet([("msisdn", "01911111111")], "suspected", 0.6, T0) + receipt = bkash.publish(T0, watch_confidence=0.6) + assert receipt["listings"] == 2 and receipt["watch"] == 1 + + bundle = hub.bundles["bkash"].to_dict() + text = str(bundle) + assert MULE[1:] not in text and "handset-77" not in text # nothing raw leaves bkash + + nagad.sync(T0 + timedelta(hours=1)) + score, matches = nagad.lookup([("msisdn", MULE_PHONE)], T0 + timedelta(hours=2)) + assert [x.status for x in matches] == ["confirmed"] + assert matches[0].typology == "impersonation" and score == pytest.approx(0.9) + score, matches = nagad.lookup([("msisdn", "01911111111")], T0 + timedelta(hours=2)) + assert [x.status for x in matches] == ["suspected"] and matches[0].listing_id is None + score, matches = nagad.lookup([("msisdn", CUSTOMER)], T0 + timedelta(hours=2)) + assert score == 0.0 and matches == [] + # The lookup is audited without the number. + assert MULE_PHONE not in str(nagad.audit.entries) and nagad.audit.verify() + + +def test_listings_expire(world): + hub, m = world + m["bkash"].list_wallet([("msisdn", MULE)], "confirmed", 0.9, T0) + m["bkash"].publish(T0, 0.6) + m["nagad"].sync(T0) + late = T0 + timedelta(days=181) + # The bundle itself is stale by then, and the listing's own TTL has passed. + assert m["nagad"].lookup([("msisdn", MULE)], late)[1] == [] + + +def test_import_rejects_bad_signature_and_replay(world): + hub, m = world + m["bkash"].list_wallet([("msisdn", MULE)], "confirmed", 0.9, T0) + bundle = m["bkash"].export_bundle(T0, 0.6) + forged = Bundle.from_dict({**bundle.to_dict(), "seq": 99}) + with pytest.raises(ProtocolError, match="signature"): + hub.submit(forged, T0) + hub.submit(bundle, T0) + with pytest.raises(ProtocolError, match="newer"): + hub.submit(bundle, T0) + # A partner checks the provider's own signature, so the hub cannot alter a listing. + tampered = Bundle.from_dict(bundle.to_dict()) + tampered.listings[0] = type(tampered.listings[0])( + **{**tampered.listings[0].__dict__, "confidence": 1.0} + ) + with pytest.raises(ProtocolError, match="signature"): + m["nagad"].ledger.import_bundle(tampered, m["bkash"].key.public, hub.epoch, T0) + + +def test_dispute_suspends_then_withdraws(world): + hub, m = world + bkash, nagad = m["bkash"], m["nagad"] + [listing_id] = bkash.list_wallet([("msisdn", CUSTOMER)], "confirmed", 0.9, T0) + bkash.publish(T0, 0.6) + nagad.sync(T0) + assert nagad.lookup([("msisdn", CUSTOMER)], T0)[0] > 0 + + dispute = hub.open_dispute("nagad", listing_id, "customer proved ownership", T0) + nagad.sync(T0 + timedelta(minutes=5)) + assert nagad.lookup([("msisdn", CUSTOMER)], T0 + timedelta(minutes=5))[0] == 0.0 + with pytest.raises(ProtocolError): + hub.resolve_dispute(dispute.dispute_id, "nagad", "withdrawn", T0) # not the owner + + hub.resolve_dispute(dispute.dispute_id, "bkash", "withdrawn", T0 + timedelta(hours=1)) + bkash.sync(T0 + timedelta(hours=1)) # the owner drops it from its own list + receipt = bkash.publish(T0 + timedelta(hours=2), 0.6) + assert receipt["listings"] == 0 and receipt["withdrawn"] == 1 + assert hub.members["bkash"].withdrawn_after_dispute == 1 + + +def test_unanswered_dispute_is_decided_for_the_customer(world): + hub, m = world + [listing_id] = m["bkash"].list_wallet([("msisdn", CUSTOMER)], "confirmed", 0.9, T0) + m["bkash"].publish(T0, 0.6) + hub.open_dispute("nagad", listing_id, "no evidence offered", T0) + assert hub.expire_disputes(T0 + DISPUTE_SLA - timedelta(minutes=1)) == [] + [late] = hub.expire_disputes(T0 + DISPUTE_SLA) + assert late.status == "withdrawn" and late.resolved_by == "hub" + + +def test_audit_chain_detects_tampering(world): + hub, m = world + m["bkash"].list_wallet([("msisdn", MULE)], "confirmed", 0.9, T0) + m["bkash"].publish(T0, 0.6) + assert hub.audit.verify() + hub.audit.entries[1]["detail"]["member"] = "someone-else" + assert not hub.audit.verify() + + +def test_signal_counts_each_provider_once(): + from fraudlens.consortium.protocol import Match + + a = Match("bkash", "msisdn", "confirmed", 0.9, "x", "bkash:1", None) + b = Match("bkash", "device", "confirmed", 0.5, "x", "bkash:2", None) + c = Match("nagad", "msisdn", "suspected", 0.5, "unknown", None, None) + assert signal([a, b]) == pytest.approx(0.9) + assert signal([a, b, c]) == pytest.approx(1 - 0.1 * 0.5) + assert signal([]) == 0.0 + + +def test_demo_state_round_trips_and_serves_lookups_and_disputes(world, tmp_path): + import json + from types import SimpleNamespace + + import pandas as pd + + from fraudlens.consortium.demo import ConsortiumService, save_demo + + hub, m = world + m["bkash"].list_wallet([("msisdn", MULE)], "confirmed", 0.9, T0, typology="impersonation") + for member in m.values(): + member.publish(T0, watch_confidence=0.6) + for member in m.values(): + member.sync(T0) + # nagad's customer W2 is on the SIM bkash listed: a cross-provider match. + t = (T0 + timedelta(hours=3)).timestamp() + send = pd.DataFrame({"t": [t], "fold": ["test"], "receiver_id": ["W2"], "mule": [0.05], + "y_mule": [1]}) # fmt: skip + token = m["nagad"].tokenize([("msisdn", MULE)], T0)[0] + found = m["nagad"].ledger.lookup([("msisdn", token)], T0) + inp = SimpleNamespace( + wallets=pd.DataFrame({"wallet_id": ["W1", "W2", "W3"], "cell_id": [3, 3, -1]}), + devices={"W2": [(t, "handset-9")]}, + threshold=0.15, + ) + market = SimpleNamespace(provider={"W1": "bkash", "W2": "nagad", "W3": "rocket"}, + msisdn={"W1": MULE, "W2": MULE, "W3": CUSTOMER}) # fmt: skip + report = {"providers": [{"name": n, "display": n.title(), "share": 0.3} for n in m]} + (tmp_path / "report.json").write_text(json.dumps(report)) + run = {"hub": hub, "members": m, "send": send, "matches": [found]} + save_demo(tmp_path, inp, market, run, hub.oprf, report) + + shared = ("bundles.json", "hub_audit.json", "matches.json") + assert not any(MULE[1:] in (tmp_path / f).read_text() for f in shared) # tokens only + + svc = ConsortiumService(tmp_path) + view = svc.overview() + assert all(f["signature_ok"] for f in view["feeds"]) and view["audit"]["chain_ok"] + [row] = svc.matches() + assert row["status"] == "confirmed" and not row["mule_model_alone_would_alert"] + assert row["simulation_truth"] == "mule" + + hit = svc.lookup("W2") + assert hit["home"] == "nagad" and hit["signal"] == pytest.approx(0.9) + assert svc.lookup("W3")["matches"] == [] + with pytest.raises(ProtocolError): + svc.lookup("W404") + + listing = hit["matches"][0]["listing_id"] + dispute = svc.open_dispute("nagad", listing, "customer appeal with evidence") + assert svc.lookup("W2")["matches"] == [] # stops counting at once + svc.resolve_dispute(dispute["dispute_id"], "upheld") + assert svc.lookup("W2")["signal"] == pytest.approx(0.9) + assert any(e["event"] == "dispute.resolved" for e in svc.audit(5)) + + +def test_dispute_route_accepts_the_listing_ids_the_hub_issues() -> None: + from pydantic import ValidationError + + from fraudlens.api.routes.consortium import DisputeIn + + hub = Hub("2026Q2", T0) + bkash = Member("bkash", hub, crypto.SigningKey.generate()) + [issued] = bkash.list_wallet([("msisdn", MULE)], "confirmed", 0.9, T0) + DisputeIn(listing_id=issued, raised_by="nagad", reason="customer appeal") + with pytest.raises(ValidationError): + DisputeIn(listing_id="bkash:x; drop", raised_by="nagad", reason="customer appeal") diff --git a/backend/tests/test_external.py b/backend/tests/test_external.py new file mode 100644 index 0000000..c1bb5c8 --- /dev/null +++ b/backend/tests/test_external.py @@ -0,0 +1,190 @@ +"""External validation helpers and the bootstrap intervals (no downloads needed).""" + +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from fraudlens.external import baf, download, paysim +from fraudlens.external.evaluate import bootstrap_ci, judge, threshold_at_fpr +from fraudlens.models.bootstrap import bootstrap, point_metrics + +# ------------------------------------------------------------------- thresholds + + +def test_threshold_at_fpr_never_exceeds_the_target(): + rng = np.random.default_rng(0) + y = (rng.random(5000) < 0.05).astype(int) + score = rng.random(5000) + y + for fpr in (0.0, 0.005, 0.01, 0.05): + thr = threshold_at_fpr(y, score, fpr) + reached = (score[y == 0] >= thr).mean() + assert reached <= fpr + assert reached >= fpr - 2 / (y == 0).sum() + + +def test_threshold_ties_are_not_alerted_past_the_budget(): + y = np.array([0, 0, 0, 0, 1]) + score = np.array([0.5, 0.5, 0.5, 0.1, 0.9]) + thr = threshold_at_fpr(y, score, 0.25) # one negative allowed, but three are tied + assert (score[y == 0] >= thr).sum() == 0 + assert score[4] >= thr + + +def test_judge_fixes_the_threshold_on_validation(): + y_val = np.array([0] * 100 + [1] * 10) + s_val = np.r_[np.linspace(0, 1, 100), np.full(10, 2.0)] + y_test = np.array([0] * 100 + [1] * 10) + s_test = s_val + 0.5 # the test scores drift up: the validation threshold over-alerts + out = judge(y_val, s_val, y_test, s_test, (0.01,)) + assert out["pr_auc"] == 1.0 + at = out["at_fpr_0.01"] + assert at["recall"] == 1.0 + assert at["fpr_on_test"] > 0.01 + assert at["recall_threshold_set_on_test"] == 1.0 + + +def test_bootstrap_ci_brackets_the_point_estimate(): + rng = np.random.default_rng(1) + y = (rng.random(3000) < 0.1).astype(int) + s = rng.random(3000) + 0.8 * y + units = np.arange(3000) // 30 + ci = bootstrap_ci(y, s, y, s, units, 0.05, replicates=100) + thr = threshold_at_fpr(y, s, 0.05) + recall = (s[y == 1] >= thr).mean() + low, high = ci["recall_at_fpr_0.05"] + assert low <= recall <= high + assert ci["units"] == 100 + + +# ------------------------------------------------------------------- PaySim features + + +def _paysim_rows(): + rows = [ + # step, type, amount, nameOrig, nameDest + (1, "CASH_IN", 100.0, "C1", "A"), + (1, "TRANSFER", 50.0, "C2", "A"), + (2, "PAYMENT", 10.0, "A", "M1"), # A sends after receiving at step 1 + (30, "TRANSFER", 200.0, "C3", "A"), + (30, "CASH_OUT", 70.0, "C4", "B"), + ] + df = pd.DataFrame(rows, columns=["step", "type", "amount", "nameOrig", "nameDest"]) + return df.assign(oldbalanceOrg=1000.0, oldbalanceDest=0.0, isFraud=0, isFlaggedFraud=0) + + +def test_paysim_features_count_only_earlier_rows(): + out = paysim.add_features(_paysim_rows()) + assert out["r_prior_in"].tolist() == [0, 1, 0, 2, 0] + # The 24-hour window at step 30 no longer sees step 1. + assert out["r_prior_in_24h"].tolist() == [0, 1, 0, 0, 0] + assert out["r_prior_in_value_24h"].iloc[3] == 0.0 + assert out["r_hours_since_last_in"].iloc[3] == 29 + assert np.isnan(out["r_hours_since_last_in"].iloc[0]) + assert out["r_prior_in_transfers"].iloc[3] == 1 + assert out["amount_vs_r_mean_in"].iloc[3] == pytest.approx(200 / 75) + # A received 50 at step 1 (the later of its two receipts) and sends at step 2. + assert out["s_hours_since_received"].iloc[2] == 1 + assert out["amount_vs_s_received"].iloc[2] == pytest.approx(10 / 50) + + +def test_paysim_features_do_not_see_the_future(): + rows = _paysim_rows() + later = pd.concat( + [rows, pd.DataFrame([{**rows.iloc[0].to_dict(), "step": 40, "amount": 9e9}])], + ignore_index=True, + ) + before = paysim.add_features(rows)[list(paysim.BEHAVIOUR_FEATURES)] + after = paysim.add_features(later)[list(paysim.BEHAVIOUR_FEATURES)].iloc[: len(rows)] + pd.testing.assert_frame_equal(before, after) + + +def test_paysim_split_is_by_time_and_scores_transfers_and_cash_outs(): + df = pd.DataFrame( + {"step": [1, 300, 301, 400, 401, 743], "type": ["TRANSFER", "CASH_OUT"] * 3} + ).assign(isFraud=0) + df.loc[0, "type"] = "PAYMENT" + folds = paysim.split(df) + assert folds["train"]["step"].tolist() == [300] + assert folds["val"]["step"].tolist() == [301, 400] + assert folds["test"]["step"].tolist() == [401, 743] + + +# ------------------------------------------------------------------- BAF + + +def test_baf_categories_come_from_the_training_months(): + df = pd.DataFrame( + { + "fraud_bool": [0, 1, 0, 0], + "month": [0, 1, 6, 7], + "device_os": ["linux", "windows", "windows", "macintosh"], + "income": [0.1, 0.2, 0.3, 0.4], + } + ) + encoded, features = baf.encode(df) + assert features == ["device_os", "income"] + assert encoded["device_os"].tolist() == [0, 1, 1, -1] # unseen in training: -1 + + +# ------------------------------------------------------------------- download + + +def test_download_refuses_a_file_with_the_wrong_checksum(tmp_path, monkeypatch): + source = download.Source("demo", "https://example.invalid/x", "x.bin", "0" * 64, 4, "") + monkeypatch.setitem(download.SOURCES, "demo", source) + monkeypatch.setattr(download, "external_dir", lambda: tmp_path) + (tmp_path / "demo").mkdir() + (tmp_path / "demo" / "x.bin").write_bytes(b"abcd") # right size: no download attempted + with pytest.raises(RuntimeError, match="does not match"): + download.fetch("demo") + good = download.sha256_of(tmp_path / "demo" / "x.bin") + monkeypatch.setitem(download.SOURCES, "demo", replace(source, sha256=good)) + assert download.fetch("demo") == tmp_path / "demo" / "x.bin" + + +# ------------------------------------------------------------------- bootstrap + + +def _test_period(): + # Day 0: scam 1 (two victim transfers, one alerted), 3 legitimate (one alerted). + # Day 1: scam 2 (one victim transfer, missed), 3 legitimate. + rows = [ + # y, y_loss, case_id, day, risk, alert + (1, 1, 1, 0, 0.9, True), + (1, 1, 1, 0, 0.2, False), + (0, 0, -1, 0, 0.8, True), + (0, 0, -1, 0, 0.1, False), + (0, 0, -1, 0, 0.1, False), + (1, 1, 2, 1, 0.3, False), + (0, 0, -1, 1, 0.05, False), + (0, 0, -1, 1, 0.05, False), + (0, 0, -1, 1, 0.05, False), + ] + df = pd.DataFrame(rows, columns=["y", "y_loss", "case_id", "day", "risk", "alert"]) + return df + + +def test_point_metrics_match_hand_counts(): + df = _test_period() + m = point_metrics( + df["y"].to_numpy(), + df["y_loss"].to_numpy() == 1, + df["case_id"].to_numpy(), + df["risk"].to_numpy(), + df["alert"].to_numpy(), + ) + assert m["scam_recall"] == 0.5 # scam 1 caught, scam 2 missed + assert m["false_alert_rate"] == pytest.approx(1 / 6) + + +def test_bootstrap_is_reproducible_and_brackets_the_point(): + df = _test_period() + a = bootstrap(df, df["risk"].to_numpy(), df["alert"].to_numpy(), replicates=200) + b = bootstrap(df, df["risk"].to_numpy(), df["alert"].to_numpy(), replicates=200) + assert a == b + assert a["n"] == {"rows": 9, "legitimate": 6, "scams": 2, "days": 2} + for scheme in ("by_day", "by_scam"): + for metric, value in a["point"].items(): + assert a[scheme][metric]["low"] <= value <= a[scheme][metric]["high"] diff --git a/backend/tests/test_ingest.py b/backend/tests/test_ingest.py new file mode 100644 index 0000000..fddd33e --- /dev/null +++ b/backend/tests/test_ingest.py @@ -0,0 +1,341 @@ +"""The signed ingest webhook, decision callbacks and token-key rotation, on Postgres and Redis. + +Needs the containers from docker-compose (`make up`). Uses its own database +(`fraudlens_test_ingest`) and Redis database 14. +""" + +from __future__ import annotations + +import itertools +import json +import shutil +from datetime import timedelta + +import pytest +from fastapi.testclient import TestClient +from redis import Redis +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from test_security import load_sdk + +from fraudlens.api import create_app +from fraudlens.config import Settings +from fraudlens.decision import evaluate, insights +from fraudlens.platform import callbacks +from fraudlens.platform.db import make_engine, make_sessions, migrate +from fraudlens.platform.events import moment +from fraudlens.platform.keys import load_keyring, main, rotate_jwt, rotate_partner +from fraudlens.platform.load import load +from fraudlens.platform.models import Transaction +from fraudlens.platform.seed import seed_users + +pytestmark = pytest.mark.integration + +PASSWORD = "correct-horse-battery-staple" +TEST_DB = "fraudlens_test_ingest" +PATH = "/v1/ingest/transactions" +CALLBACK = "http://partner.test/fraudlens/decisions?v=1" + + +@pytest.fixture(scope="module") +def keyrings(tmp_path_factory): + root = tmp_path_factory.mktemp("keys") + rotate_jwt(root / "jwt.json", 480) + rotate_partner(root / "ingest.json", "upay", timedelta(hours=24), CALLBACK) + rotate_partner(root / "ingest.json", "quietbank", timedelta(hours=24)) # no callback + return root + + +@pytest.fixture(scope="module") +def settings(trained, tmp_path_factory, keyrings): + data_dir, shared_root, _ = trained + base = Settings() + admin_url = make_url(base.database_url) + redis_url = base.redis_url.rsplit("/", 1)[0] + "/14" + try: + admin = create_engine(admin_url, isolation_level="AUTOCOMMIT") + with admin.connect() as connection: + connection.execute(text(f"DROP DATABASE IF EXISTS {TEST_DB} WITH (FORCE)")) + connection.execute(text(f"CREATE DATABASE {TEST_DB}")) + admin.dispose() + Redis.from_url(redis_url).flushdb() + except Exception as exc: # the containers are not running + pytest.skip(f"Postgres/Redis not reachable: {type(exc).__name__}") + models_root = tmp_path_factory.mktemp("ingest") / "models" + shutil.copytree(shared_root, models_root) + evaluate.run(data_dir, models_root=models_root) + insights.run(data_dir, models_root=models_root) + return Settings( + data_dir=data_dir.parent, + dataset=data_dir.name, + models_root=models_root, + database_url=admin_url.set(database=TEST_DB).render_as_string(hide_password=False), + redis_url=redis_url, + events_stream="fraudlens-ingest-test:events", + alerts_channel="fraudlens-ingest-test:alerts", + run_worker=False, + seed_password=PASSWORD, + llm_notes=False, + jwt_keyring=keyrings / "jwt.json", + ingest_keyring=keyrings / "ingest.json", + ) + + +@pytest.fixture(scope="module") +def client(settings): + migrate(settings.database_url) + engine = make_engine(settings.database_url) + redis = Redis.from_url(settings.redis_url, decode_responses=True) + with make_sessions(engine)() as session: + seed_users(session, settings) + load(engine, redis, settings, reset=True) + engine.dispose() + with TestClient(create_app(settings)) as test_client: + yield test_client + + +@pytest.fixture(scope="module") +def p(client): + return client.app.state.platform + + +@pytest.fixture(scope="module") +def sdk(): + return load_sdk() + + +@pytest.fixture(scope="module") +def wallets(p): + engine = p.scorer.engine + pool = iter( + w for w, state in sorted(engine.wallets.items()) if w not in engine.flagged and state.home + ) + return lambda: next(pool) + + +_ids = itertools.count(9_100_000_000) + + +def transfer(sdk, p, sender: str, receiver: str, amount: float = 900.0, **over) -> dict: + txn_id = next(_ids) + state = p.scorer.engine.wallets[sender] + args = { + "txn_id": txn_id, + "end_to_end_id": f"E2E-{txn_id}", + "accepted_at": moment(p.scorer.clock.now()), + "purpose": "SEND_MONEY", + "amount": amount, + "debtor": (sender, "wallet"), + "creditor": (receiver, "wallet"), + "channel": "app", + "district": state.home, + "debtor_balance_before": 20_000.0, + "device_id": next(iter(state.devices), ""), + } + return sdk.credit_transfer(**(args | over)) + + +def key_of(settings, partner: str = "upay"): + return load_keyring(settings.ingest_keyring).signing(partner) + + +def post(client, sdk, settings, message, path=PATH, partner="upay", idem=None, **header_over): + body = json.dumps(message).encode() + key = key_of(settings, partner) + headers = sdk.signed_headers(key.kid, key.secret, "POST", path, body) + if idem: + headers["Idempotency-Key"] = idem + headers.update(header_over) + return client.post(path, content=body, headers=headers), headers, body + + +def code(response) -> str: + return response.json()["error"]["code"] + + +# ---------------------------------------------------------------- authentication + + +def test_unsigned_and_badly_signed_requests_are_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-A", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + assert code(client.post(PATH, content=body)) == "unsigned" + response, headers, _ = post(client, sdk, settings, message) + assert response.status_code == 202, response.text + # The same signature on a body with one figure changed. + tampered = body.replace(b'"Amt": 900.0', b'"Amt": 9000.0') + assert tampered != body + forged = client.post(PATH, content=tampered, headers=headers | {"X-FraudLens-Nonce": "n" * 20}) + assert (forged.status_code, code(forged)) == (401, "bad_signature") + # Signed for the queue, sent asking for a decision. + wrong_path = client.post(PATH + "?wait=decision", content=body, headers=headers) + assert code(wrong_path) == "bad_signature" + + +def test_a_stale_timestamp_is_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-B", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + key = key_of(settings) + import time + + for skew in (-301, 301): + headers = sdk.signed_headers(key.kid, key.secret, "POST", PATH, body, time.time() + skew) + response = client.post(PATH, content=body, headers=headers) + assert (response.status_code, code(response)) == (401, "stale_timestamp") + + +def test_a_replayed_request_is_refused(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-C", [transfer(sdk, p, wallets(), wallets())]) + first, headers, body = post(client, sdk, settings, message) + assert first.status_code == 202 + again = client.post(PATH, content=body, headers=headers) + assert (again.status_code, code(again)) == (409, "replayed") + + +def test_a_retired_or_unknown_key_is_refused(client, sdk, settings, p, wallets): + old = key_of(settings, "quietbank") + rotate_partner(settings.ingest_keyring, "quietbank", timedelta(hours=1)) + message = sdk.pacs008("MSG-D", [transfer(sdk, p, wallets(), wallets())]) + body = json.dumps(message).encode() + # During the grace period the old key still works. + headers = sdk.signed_headers(old.kid, old.secret, "POST", PATH, body) + assert client.post(PATH, content=body, headers=headers).status_code == 202 + main(["retire", str(settings.ingest_keyring), old.kid]) + headers = sdk.signed_headers(old.kid, old.secret, "POST", PATH, body) + response = client.post(PATH, content=body, headers=headers) + assert (response.status_code, code(response)) == (401, "unknown_key") + + +# ---------------------------------------------------------------- what comes in + + +def test_a_decision_can_be_had_synchronously(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-E", [transfer(sdk, p, wallets(), wallets())]) + response, _, _ = post(client, sdk, settings, message, path=PATH + "?wait=decision") + assert response.status_code == 200, response.text + answer = response.json() + txn = message["CdtTrfTxInf"][0] + assert answer["end_to_end_id"] == txn["PmtId"]["EndToEndId"] + assert answer["txn_id"] == int(txn["PmtId"]["TxId"]) and answer["scored"] + assert answer["decision"]["tier"] in {"allow", "warn", "step_up", "hold"} + # Queued again under another message: recognised, not scored twice. + again = sdk.pacs008("MSG-E2", message["CdtTrfTxInf"]) + response, _, _ = post(client, sdk, settings, again) + assert response.json()["accepted"] == 0 + assert response.json()["duplicates"] == [txn["PmtId"]["EndToEndId"]] + + +def test_an_idempotency_key_returns_the_first_answer(client, sdk, settings, p, wallets): + message = sdk.pacs008("MSG-F", [transfer(sdk, p, wallets(), wallets())]) + path = PATH + "?wait=decision" + first, _, _ = post(client, sdk, settings, message, path=path, idem="retry-F") + retry, _, _ = post(client, sdk, settings, message, path=path, idem="retry-F") # new nonce + assert first.status_code == retry.status_code == 200 + assert retry.json() == first.json() and not retry.json()["duplicate"] + assert retry.headers["Idempotent-Replayed"] == "true" + other = sdk.pacs008("MSG-F2", [transfer(sdk, p, wallets(), wallets())]) + reused, _, _ = post(client, sdk, settings, other, path=path, idem="retry-F") + assert (reused.status_code, code(reused)) == (422, "idempotency_key_reused") + + +def test_a_batch_is_queued_in_order_and_scored_by_the_stream(client, sdk, settings, p, wallets): + transfers = [transfer(sdk, p, wallets(), wallets()) for _ in range(3)] + transfers[1]["PmtTpInf"]["CtgyPurp"]["Prtry"] = "CASH_IN" # an agent pays a wallet + transfers[1]["DbtrAcct"]["Id"]["Othr"] = {"Id": "AG_NOBODY", "SchmeNm": {"Prtry": "agent"}} + transfers[1]["SplmtryData"]["Envlp"]["Chanl"] = "agent" + message = sdk.pacs008("MSG-G", transfers) + single, _, _ = post(client, sdk, settings, message) + assert code(single) == "invalid_request" # one endpoint, one transaction + response, _, _ = post(client, sdk, settings, message, path=PATH + "/batch") + assert response.status_code == 202, response.text + assert response.json()["accepted"] == 3 and response.json()["callback"] + p.worker.drain() + with p.sessions() as s: + stored = [s.get(Transaction, int(t["PmtId"]["TxId"])) for t in transfers] + assert all(t is not None and t.source == "live" for t in stored) + + +def test_schema_problems_name_the_iso_element(client, sdk, settings, p, wallets): + bad = transfer(sdk, p, wallets(), wallets()) + bad["IntrBkSttlmAmt"] = {"Ccy": "USD", "Amt": -5} + wrong_kind = transfer(sdk, p, wallets(), wallets()) + wrong_kind["CdtrAcct"]["Id"]["Othr"]["SchmeNm"]["Prtry"] = "agent" # SEND_MONEY to an agent + for message, where in [ + (sdk.pacs008("MSG-H", [bad]), "CdtTrfTxInf.0.IntrBkSttlmAmt"), + (sdk.pacs008("MSG-I", [wrong_kind]), "CdtTrfTxInf.0"), + ]: + response, _, _ = post(client, sdk, settings, message) + assert response.status_code == 422 + fields = [f["field"] for f in response.json()["error"]["fields"]] + assert all(f.startswith(where) for f in fields), fields + assert "-5" not in response.text # the value sent is not repeated + + +# ---------------------------------------------------------------- callbacks + + +def test_the_decision_is_called_back_signed_with_retries(client, sdk, settings, p, wallets): + received, statuses = [], iter([503, 204]) + + def fake_post(url: str, headers: dict, body: bytes) -> int: + received.append((url, headers, body)) + return next(statuses) + + dispatcher = callbacks.Dispatcher(settings, p.sessions, p.redis, post=fake_post) + p.redis.delete(callbacks.SCHEDULE) + message = sdk.pacs008("MSG-J", [transfer(sdk, p, wallets(), wallets())]) + response, _, _ = post(client, sdk, settings, message) + assert response.status_code == 202 + import time + + now = time.time() + # Not scored yet: the job waits and nothing is sent. + assert dispatcher.run_once(now + 3) == 1 and received == [] + p.worker.drain() + assert dispatcher.run_once(now + 6) == 1 and len(received) == 1 # 503: retried later + [(member, due)] = p.redis.zrange(callbacks.SCHEDULE, 0, -1, withscores=True) + assert json.loads(member)["try"] == 1 and now + 6 + 1.5 <= due <= now + 6 + 5 + assert dispatcher.run_once(now + 6) == 0 # not due yet + assert dispatcher.run_once(due) == 1 and len(received) == 2 + assert p.redis.zcard(callbacks.SCHEDULE) == 0 and dispatcher.delivered == 1 + + url, headers, body = received[-1] + assert url == CALLBACK + key = key_of(settings) + assert sdk.verify({key.kid: key.secret}, headers, "POST", "/fraudlens/decisions?v=1", body) + payload = json.loads(body) + txn = message["CdtTrfTxInf"][0] + assert payload["end_to_end_id"] == txn["PmtId"]["EndToEndId"] + assert payload["decision"]["tier"] in {"allow", "warn", "step_up", "hold"} + assert "reasons" not in payload["decision"] # what to do, not why + + +def test_a_callback_that_keeps_failing_ends_in_the_dead_list(client, sdk, settings, p): + dispatcher = callbacks.Dispatcher(settings, p.sessions, p.redis, post=lambda *a: 500) + p.redis.delete(callbacks.SCHEDULE, callbacks.DEAD) + callbacks.schedule(p.redis, "upay", 2**61, "E2E-NEVER", now=0.0) # never arrives + now = callbacks.DECISION_WAIT + 10 # long enough: reported as not processed + for _ in range(callbacks.MAX_ATTEMPTS): + now += callbacks.MAX_DELAY * 1.3 + assert dispatcher.run_once(now) == 1 + assert p.redis.zcard(callbacks.SCHEDULE) == 0 + [dead] = p.redis.lrange(callbacks.DEAD, 0, -1) + assert json.loads(dead)["payload"]["status"] == "not_processed" + + +# ---------------------------------------------------------------- token keys + + +def test_sessions_survive_a_signing_key_rotation(client, settings): + login = client.post("/v1/auth/login", json={"username": "analyst1", "password": PASSWORD}) + assert login.status_code == 200, login.text + old = {"Authorization": f"Bearer {login.json()['access_token']}"} + import jwt + + old_kid = jwt.get_unverified_header(login.json()["access_token"])["kid"] + rotate_jwt(settings.jwt_keyring, settings.jwt_ttl_minutes) # no restart + again = client.post("/v1/auth/login", json={"username": "analyst1", "password": PASSWORD}) + assert jwt.get_unverified_header(again.json()["access_token"])["kid"] != old_kid + assert client.get("/v1/auth/me", headers=old).status_code == 200 + main(["retire", str(settings.jwt_keyring), old_kid]) + assert client.get("/v1/auth/me", headers=old).status_code == 401 diff --git a/backend/tests/test_intel.py b/backend/tests/test_intel.py index 9a94003..ddeb9bc 100644 --- a/backend/tests/test_intel.py +++ b/backend/tests/test_intel.py @@ -246,3 +246,112 @@ def test_a_customer_report_is_labelled_from_the_choice_and_the_description(model read = categorise_report("other", OTP_SCAM, model, TAXONOMY) assert read and all(c["basis"] == ["description"] for c in read) assert categorise_report("other", LUNCH, None, TAXONOMY) == [] + + +# ------------------------------------------------- cue-level reasons, both scripts + + +@pytest.mark.parametrize( + "text, cue, phrase", + [ + (OTP_SCAM, "ask_secret", "OTP code ta bolun"), + ("আপনার অ্যাকাউন্ট বন্ধ হয়ে যাবে, এখনই ওটিপি কোডটি বলুন।", "ask_secret", "ওটিপি কোডটি বলুন"), + ("Sir আপনার account block হবে, OTP টা এখনই বলুন।", "ask_secret", "OTP টা এখনই বলুন"), + ("Prize পেতে ৫০০ টাকা processing fee পাঠান।", "pay_first", "fee পাঠান"), + ], +) +def test_a_reason_points_at_the_words_that_showed_it(text, cue, phrase): + from fraudlens.intel.explain import cue_phrases, mapped + + assert mapped(text)[0] == normalise(text) # the offsets follow the same normalisation + found = {c.id: phrases for c, phrases in cue_phrases(text)} + assert cue in found + spans = found[cue] + assert any(phrase in p["text"] for p in spans), spans + assert all(text[p["start"] : p["end"]].strip() == p["text"] for p in spans) + + +def test_a_flagged_message_carries_its_phrases_and_highlighted_words(model): + found = check_message(OTP_SCAM, model, TAXONOMY) + secret = next(s for s in found["signals"] if s["id"] == "ask_secret") + assert secret["label"]["bn"] and secret["phrases"][0]["text"] + words = found["highlights"] + assert words and all(OTP_SCAM[w["start"] : w["end"]] == w["text"] for w in words) + assert words == sorted(words, key=lambda w: -w["weight"]) + assert check_message(LUNCH, model, TAXONOMY)["highlights"] == [] + + +# -------------------------------------------------------------- public corpus + + +def test_the_public_corpus_is_used_only_when_it_is_the_pinned_one(tmp_path): + from fraudlens.intel import external + + assert external.load(tmp_path) is None # nothing there: trained on the corpus alone + (tmp_path / "train.parquet").write_bytes(b"not the pinned file") + assert external.load(tmp_path) is None + meta = external.provenance() + assert meta["licence"] == "MIT" and len(meta["revision"]) == 40 + assert all(len(f["sha256"]) == 64 for f in meta["files"].values()) + + +def test_every_scam_family_has_a_typology_and_each_is_held_out_once(): + families = corpus.load_families() + scams = [f for f in families if f.labels] + assert all(f.typology in corpus.TYPOLOGIES for f in scams) + assert all(f.typology is None for f in families if not f.labels) + held = {f.typology for f in scams if f.held_out} + assert set(corpus.TYPOLOGIES) - {"other"} <= held + + +# ------------------------------------------------------- message, then payment + + +def test_a_payment_that_follows_a_flagged_message_is_linked_to_it(): + from datetime import timedelta + + from fraudlens.intel.memory import MessageMemory, targets + + wallets, phones, amounts = targets("ভুল করে ৳২,৫০০ গেছে, W0000331 নম্বরে ফেরত দিন বা 01712345678") + assert wallets == {"W0000331"} and phones == {"01712345678"} and amounts == (2500.0,) + + memory, now = MessageMemory(), datetime(2026, 1, 1, 12, tzinfo=UTC) + flagged = {"level": "high", "categories": [{"id": "social_engineering"}]} + text = "Bhul kore 2,500 taka gese, W0000331 e ferot pathan" + assert not memory.remember("W1", text, {"level": "none"}, now) # not flagged: not kept + assert memory.remember("W1", text, flagged, now) + later = now + timedelta(minutes=10) + assert memory.link("W1", "W0000331", 100.0, later)["matched"] == ["receiver"] + assert memory.link("W1", "W0000999", 2500.0, later)["matched"] == ["amount"] + assert memory.link("W1", "W0000999", 100.0, later) is None + assert memory.link("W2", "W0000331", 2500.0, later) is None # someone else's message + assert memory.link("W1", "W0000331", 2500.0, now + timedelta(minutes=31)) is None + + +def test_a_linked_payment_is_warned_with_the_reason_in_both_languages(): + from dataclasses import dataclass, field + + from fraudlens.decision import load_policy + from fraudlens.intel.memory import escalate + + @dataclass(frozen=True) + class Decision: + tier: str = "allow" + action: str = "proceed" + scenario: str | None = None + customer_message: dict | None = None + decided_by: str = "model" + reasons: tuple = () + evidence: dict = field(default_factory=dict) + + policy = load_policy(Settings().policy_version) + link = {"matched": ["receiver", "amount"], "minutes_ago": 4, "level": "high", "categories": []} + warned = escalate(Decision(), link, policy) + assert warned.tier == "warn" and warned.scenario == "scam" + assert warned.action == policy.tiers["warn"].action and warned.customer_message["bn"] + reason = warned.reasons[0] + assert reason["code"] == "follows_flagged_message" and reason["direction"] == "raises" + assert "4 minutes" in reason["detail_en"] and reason["detail_bn"] + assert warned.evidence["message_link"] == link + held = escalate(Decision(tier="hold", action="hold_for_review"), link, policy) + assert held.tier == "hold" and held.reasons[0]["code"] == "follows_flagged_message" diff --git a/backend/tests/test_label_realism.py b/backend/tests/test_label_realism.py new file mode 100644 index 0000000..9f14e5c --- /dev/null +++ b/backend/tests/test_label_realism.py @@ -0,0 +1,103 @@ +import numpy as np +import pandas as pd +import pytest + +from fraudlens.models import label_realism as L + + +def _cases(n: int = 400) -> pd.DataFrame: + rng = np.random.default_rng(0) + typologies = ["account_takeover", "lottery_fee", "wrong_send", "impersonation"] + return pd.DataFrame( + { + # Case ids are not positions: some cases never produced a transfer. + "case_id": np.arange(n) * 3 + 5, + "typology": [typologies[i % 4] for i in range(n)], + "loss": rng.lognormal(8, 1, n), + "urban": (rng.random(n) < 0.5).astype(float), + "tenure_days": rng.uniform(10, 900, n), + "started_at": pd.Timestamp("2026-01-01") + pd.to_timedelta(np.arange(n), unit="h"), + } + ) + + +def test_biased_reporting_averages_the_simulator_rate_and_favours_takeovers(): + cases = _cases() + p = np.asarray(L.report_probability(cases, 0.5)) + assert p.mean() == pytest.approx(0.5, abs=1e-6) + by_type = pd.Series(p).groupby(cases["typology"]).mean() + assert by_type["account_takeover"] > by_type["impersonation"] > by_type["lottery_fee"] + big = cases["loss"] > cases["loss"].median() + assert p[big].mean() > p[~big].mean() + assert np.all(np.asarray(L.report_probability(cases, 0.5, biased=False)) == 0.5) + + +def test_reports_stay_with_their_case_when_case_ids_are_not_positions(): + cases = _cases() + p = np.where(cases["typology"] == "account_takeover", 1.0, 0.0) + at = L.draw_reports(cases, p, np.random.default_rng(1)) + assert list(at.index) == list(cases["case_id"]) + reported = at.notna().to_numpy() + assert np.array_equal(reported, cases["typology"].to_numpy() == "account_takeover") + assert (at.dropna() > cases.set_index("case_id")["started_at"].reindex(at.dropna().index)).all() + + +def test_only_reports_in_by_the_cutoff_label_their_case(): + ts = pd.Timestamp("2026-02-01") + report_at = pd.Series([ts, ts + pd.Timedelta(days=5), pd.NaT], index=[10, 11, 12]) + frame = pd.DataFrame({"y": [1, 1, 1, 0], "case_id": [10, 11, 12, -1]}) + y = L.observed_labels(frame, report_at, ts + pd.Timedelta(days=1)) + assert y.tolist() == [True, False, False, False] + + +def test_propagation_reaches_other_money_through_a_reported_mule_within_the_window(): + t = pd.Timestamp("2026-02-01") + frame = pd.DataFrame( + { + "type": ["SEND_MONEY", "SEND_MONEY", "CASH_OUT", "SEND_MONEY", "SEND_MONEY"], + "sender_id": ["V1", "V2", "M1", "V3", "X"], + "receiver_id": ["M1", "M1", "A1", "M1", "Y"], + "ts": [t, t + pd.Timedelta(hours=5), t + pd.Timedelta(hours=6), + t + pd.Timedelta(days=10), t], + } + ) # fmt: skip + y_obs = np.array([True, False, False, False, False]) + hit = L.propagate(frame, y_obs) + # Another victim's transfer and the mule's cash-out; not the one 10 days later, + # not unrelated traffic, and never the already-labelled row. + assert hit.tolist() == [False, True, True, False, False] + + +def test_elkan_noto_weights(): + g = np.array([0.4, 0.4, 0.2, 0.01]) + s = np.array([1.0, 1.0, 0.0, 0.0]) + w, c = L.elkan_noto(g, s) + assert c == pytest.approx(0.4) + assert np.isnan(w[:2]).all() + assert w[2] == pytest.approx(1.5 * 0.2 / 0.8) + assert 0 <= w[3] < w[2] <= 1 + w_known, c_known = L.elkan_noto(g, s, 0.5) + assert c_known == 0.5 and w_known[2] == pytest.approx(0.25) + + +def test_soft_rows_are_split_into_weighted_fraud_and_clean_copies(): + x = np.arange(6, dtype=float).reshape(3, 2) + y = np.array([1.0, 0.0, 0.0]) + xs, ys, ws = L.with_soft_labels(x, y, np.array([np.nan, 0.7, np.nan])) + assert len(xs) == 4 and ys.tolist() == [1.0, 0.0, 1.0, 0.0] + assert ws.tolist() == pytest.approx([1.0, 1.0, 0.7, 0.3]) + assert L.with_soft_labels(x, y, None)[2] is None + + +def test_experiment_runs_every_regime_at_the_served_false_positive_rate(trained): + data_dir, models_root, _ = trained + report = L.run(data_dir, models_root, seeds=(7,)) + assert list(report["summary"]["regimes"]) == list(L.REGIMES) + target = report["operating_point"]["fpr"] + for result in report["runs"]["7"]["regimes"].values(): + assert result["fpr"] <= target + 1e-3 + assert 0 <= result["pr_auc"] <= 1 + labels = report["runs"]["7"]["labels"] + assert labels["ground_truth"]["case_label_rate"] == 1.0 + assert labels["reported"]["fraud_rows_labelled"] < labels["ground_truth"]["fraud_rows"] + assert labels["reported"]["clean_rows_labelled_fraud"] == 0 diff --git a/backend/tests/test_mitigation.py b/backend/tests/test_mitigation.py new file mode 100644 index 0000000..a057a0f --- /dev/null +++ b/backend/tests/test_mitigation.py @@ -0,0 +1,241 @@ +"""Segment thresholds for young wallets: the guards, the decision path and the fit.""" + +import json + +import numpy as np +import pandas as pd +import pytest +import yaml + +from fraudlens.decision import PolicyError, apply_policy, load_policy +from fraudlens.decision import mitigation as mit +from fraudlens.decision.policy import POLICY_DIR, Policy + +BASE = {"warn": 0.002, "step_up": 0.008, "hold": 0.05} +MANIFEST = {"thresholds": BASE} + + +def _segment(**changes) -> dict: + base = { + "id": "S01_YOUNG_RECEIVER", + "description": "test segment", + "applies_to": ["SEND_MONEY"], + "when": [{"field": "r_age_days", "op": "<", "value": 30}], + "scale": {"warn": 10, "step_up": 2, "hold": 2}, + } + return base | changes + + +def _policy(*segments: dict) -> Policy: + raw = yaml.safe_load((POLICY_DIR / "v2.yaml").read_text(encoding="utf-8")) + return Policy.model_validate(raw | {"segments": list(segments)}) + + +# ------------------------------------------------------------------ the guards + + +@pytest.mark.parametrize( + "scale", + [ + {"warn": 10, "step_up": 2, "hold": 0.1}, # hold below warn + ], +) +def test_an_unsafe_segment_is_refused_when_the_thresholds_are_resolved(scale): + policy = _policy(_segment(scale=scale)) + with pytest.raises(PolicyError, match="segment S01_YOUNG_RECEIVER: thresholds must satisfy"): + policy.resolve_thresholds(MANIFEST) + + +def test_a_payment_held_for_anyone_is_at_least_warned_in_every_segment(): + # warn x30 would be 0.06, above the policy's hold cut-off: capped there, and + # step-up kept between warn and the segment's hold. + segment = _policy(_segment(scale={"warn": 30, "step_up": 1, "hold": 3})).segments[0] + assert segment.thresholds(BASE) == pytest.approx({"warn": 0.05, "step_up": 0.05, "hold": 0.15}) + # A hold scale that would pass 1 keeps 1/scale of the room below 1 instead. + high = {"warn": 0.99, "step_up": 0.995, "hold": 0.999} + assert segment.thresholds(high)["hold"] == pytest.approx(1 - 0.001 / 3) + + +@pytest.mark.parametrize( + "changes, message", + [ + ({"scale": {"warn": 10, "hold": 2}}, "scale needs each of"), + ({"scale": {"warn": 0, "step_up": 2, "hold": 2}}, "positive"), + ({"applies_to": ["PAY_BILL"]}, "unknown transaction types"), + ({"when": []}, "at least 1"), + ({"id": "lower"}, "pattern"), + ], +) +def test_segment_validation_rejects_bad_entries(changes, message): + with pytest.raises(ValueError, match=message): + _policy(_segment(**changes)) + + +def test_segment_ids_must_be_unique(): + with pytest.raises(ValueError, match="segment"): + _policy(_segment(), _segment()) + + +# ------------------------------------------------------------ the decision path + +YOUNG = {"r_age_days": 3.0, "s_age_days": 400.0, "recipient_flagged": 0.0, "sender_flagged": 0.0} + + +def test_a_segment_applies_its_own_cut_offs_and_says_so(): + policy = _policy(_segment()) + thresholds = policy.resolve_thresholds(MANIFEST) + risk = 0.01 # step-up for anyone; below this segment's warn cut-off (0.02) + plain = apply_policy(policy, "SEND_MONEY", YOUNG | {"r_age_days": 300.0}, risk, thresholds) + young = apply_policy(policy, "SEND_MONEY", YOUNG, risk, thresholds) + assert (plain.tier, plain.segment) == ("step_up", None) + assert young.tier == "allow" and young.segment.id == "S01_YOUNG_RECEIVER" + assert young.thresholds["warn"] == pytest.approx(0.02) + # A cash-out has no receiving wallet: the segment does not apply. + assert apply_policy(policy, "CASH_OUT", YOUNG, risk, thresholds).segment is None + + +def test_the_softer_tier_steps_up_instead_of_holding_but_still_holds_above_it(): + policy = _policy(_segment()) # hold cut-off 0.1 for the segment, 0.05 for anyone + thresholds = policy.resolve_thresholds(MANIFEST) + assert apply_policy(policy, "SEND_MONEY", YOUNG, 0.07, thresholds).tier == "step_up" + assert apply_policy(policy, "SEND_MONEY", YOUNG, 0.2, thresholds).tier == "hold" + + +def test_the_first_matching_segment_decides(): + second = _segment(id="S02_YOUNG_SENDER", applies_to=["SEND_MONEY", "CASH_OUT"]) + second["when"] = [{"field": "s_age_days", "op": "<", "value": 30}] + policy = _policy(_segment(), second) + both = YOUNG | {"s_age_days": 2.0} + assert policy.segment_for("SEND_MONEY", both).id == "S01_YOUNG_RECEIVER" + assert policy.segment_for("CASH_OUT", both).id == "S02_YOUNG_SENDER" + assert policy.segment_for("SEND_MONEY", YOUNG | {"r_age_days": float("nan")}) is None + + +def test_hard_rules_still_hold_inside_a_segment(): + policy = _policy(_segment()) + thresholds = policy.resolve_thresholds(MANIFEST) + flagged = YOUNG | {"recipient_flagged": 1.0} + outcome = apply_policy(policy, "SEND_MONEY", flagged, 0.0001, thresholds) + assert outcome.tier == "hold" and outcome.segment is not None + assert policy.tiers["hold"].human_review + + +def test_rules_only_mode_ignores_segments(): + policy = _policy(_segment()) + outcome = apply_policy(policy, "SEND_MONEY", YOUNG, None, None) + assert outcome.mode == "rules_only" and outcome.segment is None + + +def test_v3_is_v2_plus_segments_and_resolves_against_the_served_model_kind(): + v2, v3 = load_policy("v2"), load_policy("v3") + assert v3.version == "v3" and [s.id for s in v3.segments] == [ + "S01_YOUNG_RECEIVER", + "S02_YOUNG_SENDER", + ] + assert v3.rules == v2.rules and v3.tiers == v2.tiers and not v2.segments + for segment in v3.segments: # admissible for the thresholds it was fitted on + assert segment.fitted + segment.thresholds(BASE | {"warn": 0.0019854631219750926, "hold": 0.04978453405976952}) + + +def test_v3_decisions_record_the_segment(world_v3): + rows = world_v3.frame + young = rows[(rows["type"] == "SEND_MONEY") & (rows["r_age_days"] < 30)].iloc[0] + decision = world_v3.decide(young) + assert decision.segment["id"] == "S01_YOUNG_RECEIVER" + assert set(decision.segment["thresholds"]) == {"warn", "step_up", "hold"} + assert decision.policy_version == "v3" + + +@pytest.fixture(scope="module") +def world_v3(trained): + from types import SimpleNamespace + + from fraudlens.decision import DecisionEngine + from fraudlens.decision import evaluate as policy_eval + from fraudlens.features import FEATURES + from fraudlens.features.build import iter_txns + from fraudlens.models import registry + from fraudlens.models.data import load_frame + + data_dir, root, _ = trained + bundle = registry.load(root=root) + frame = policy_eval.add_context( + load_frame(data_dir), pd.read_parquet(data_dir / "wallet_flags.parquet") + ) + txns = pd.read_parquet(data_dir / "transactions.parquet").set_index("txn_id") + engine = DecisionEngine(load_policy("v3"), bundle) + + def decide(row): + txn = next(iter_txns(txns.loc[[row["txn_id"]]].reset_index())) + context = {k: row[k] for k in policy_eval.CONTEXT_COLUMNS} + return engine.decide(txn, row[list(FEATURES)].to_numpy(float), context) + + return SimpleNamespace(frame=frame, decide=decide) + + +# --------------------------------------------------------------------- the fit + + +def test_scales_keep_step_up_between_warn_and_hold_and_respect_the_guard(): + for warn in mit.WARN_SCALES: + for hold in mit.HOLD_SCALES: + scale = mit.scales(BASE, warn, hold) + t = {k: BASE[k] * v for k, v in scale.items()} + if mit.admissible(BASE, scale): + assert t["warn"] <= t["step_up"] <= t["hold"] <= BASE["hold"] * hold + assert t["warn"] <= BASE["hold"] + else: + assert t["warn"] > BASE["hold"] or t["hold"] >= 1 + + +def test_the_fit_picks_the_smallest_gap_within_the_recall_budget(): + # Two options per segment, one segment: base (1, 1) and a cheaper but lossy one. + options = [(1, 1), (2, 1), (4, 1)] + names = ["largest_young_ratio_ref", "summed_young_ratio_ref", "largest_young_hold_ratio"] + fit = { + "largest_young_ratio_ref": np.array([10.0, 5.0, 2.0]), + "summed_young_ratio_ref": np.array([12.0, 6.0, 3.0]), + "largest_young_hold_ratio": np.array([3.0, 3.0, 3.0]), + "victim_recall": np.array([0.9, 0.898, 0.8]), # (4, 1) loses 10 points + "taka_recall": np.array([0.9, 0.9, 0.9]), + "victim_recall_hold": np.array([0.8, 0.8, 0.8]), + } + for key in mit._FRONTIER_KEYS: + fit.setdefault(key, np.zeros(3)) + assert set(names) <= set(fit) + out = mit.fit(fit, fit, options, BASE, ["S01"]) + assert out["chosen"]["scales"] == {"S01": {"warn": 2, "hold": 1}} + assert out["base"]["scales"] == {"S01": {"warn": 1, "hold": 1}} + assert out["meeting_constraints"] == 2 + + +def test_metrics_measure_young_wallet_rates_against_the_right_rows(): + rows = pd.DataFrame( + { + "type": ["SEND_MONEY", "SEND_MONEY", "CASH_OUT", "SEND_MONEY"], + "s_age_days": [400.0, 400.0, 5.0, 400.0], + "r_age_days": [5.0, 400.0, np.nan, 400.0], + "y": [0, 0, 0, 1], + "y_loss": [0, 0, 0, 1], + "amount": [100.0, 100.0, 100.0, 500.0], + } + ) + m = mit._metrics(mit._sums(mit._columns(rows, np.array([1, 0, 1, 3])))) + assert m["false_alert_rate"] == pytest.approx(2 / 3) + assert m["receiver_young_false_alert_rate"] == 1.0 + assert m["sender_young_false_alert_rate"] == 1.0 + assert m["reference_false_alert_rate"] == 0.0 + assert m["victim_recall"] == m["victim_recall_hold"] == m["taka_recall"] == 1.0 + + +def test_the_report_runs_end_to_end_and_the_engine_agrees_with_the_grid(trained): + data_dir, root, _ = trained + report = mit.run(data_dir, "v3", models_root=root, reps=20) + assert report["engine_matches_grid"] + assert report["policy_version"] == "v3" and report["fit"]["chosen"]["admissible"] + keys = {m["key"] for m in report["before_after"]["metrics"]} + assert {"receiver_young_ratio", "sender_young_ratio", "victim_recall"} <= keys + assert set(report["intersectional"]) == {"sender", "receiver"} + written = json.loads((root / report["model_version"] / mit.MITIGATION_FILE).read_text()) + assert written["policy_version"] == "v3" diff --git a/backend/tests/test_platform.py b/backend/tests/test_platform.py index d6a96cb..05ee5f6 100644 --- a/backend/tests/test_platform.py +++ b/backend/tests/test_platform.py @@ -12,6 +12,8 @@ import itertools import json import shutil +import threading +from datetime import datetime import numpy as np import pytest @@ -33,7 +35,7 @@ from fraudlens.models.train import FEEDBACK_COLUMNS from fraudlens.platform import replay, verify from fraudlens.platform.db import make_engine, make_sessions, migrate -from fraudlens.platform.events import moment, network_of +from fraudlens.platform.events import event_adapter, moment, network_of from fraudlens.platform.load import load from fraudlens.platform.models import ( AuditLog, @@ -41,15 +43,17 @@ CustomerReport, Decision, FreezeRequest, + PendingDecision, ShadowScore, Transaction, User, Wallet, WalletFlag, ) +from fraudlens.platform.scoring import SEQUENCE_LOCK, Scorer from fraudlens.platform.security import check_production from fraudlens.platform.seed import seed_users -from fraudlens.platform.stream import DEAD_SUFFIX, alert_feed +from fraudlens.platform.stream import DEAD_SUFFIX, GROUP, Worker, alert_feed pytestmark = pytest.mark.integration @@ -137,12 +141,33 @@ def headers(username: str) -> dict: @pytest.fixture(scope="module") -def replayed(client, p, settings): - """The whole test period sent through the event stream and scored.""" +def replica(client, p, settings): + """A second scorer on the same database: what a stream worker process holds.""" + return Scorer(settings, p.sessions, p.redis) + + +def catch_up(scorer: Scorer) -> None: + with scorer.lock, scorer.sessions() as s: + s.execute(SEQUENCE_LOCK) + scorer.refresh(s) + s.commit() + + +@pytest.fixture(scope="module") +def replayed(client, p, settings, replica): + """The whole test period sent through the event stream and scored by two workers + in one consumer group, each with its own copy of the state.""" events = replay.test_events(settings.dataset_dir) sent = replay.via_stream(p.redis, settings.events_stream, events, wait=False) - handled = p.worker.drain() - return {"sent": sent, "handled": handled} + other = Worker(replica, p.redis, settings, consumer="test-worker-b") + p.worker.ensure_group() + handled = {} + thread = threading.Thread(target=lambda: handled.update(b=other.drain())) + thread.start() + handled["a"] = p.worker.drain() + thread.join() + catch_up(p.scorer) # the tests below read the API's state directly + return {"sent": sent, "handled": handled["a"] + handled["b"], "workers": handled} _ids = itertools.count(9_000_000_000) @@ -217,6 +242,9 @@ def held_payment(client, tokens, p, spare) -> tuple[dict, dict]: def test_replay_through_the_stream_matches_the_offline_evaluation(p, settings, replayed): assert replayed["handled"] == replayed["sent"]["events"] > 0 + assert min(replayed["workers"].values()) > 0, replayed["workers"] # both took a share + with p.sessions() as s: + assert s.scalar(select(func.count()).select_from(PendingDecision)) == 0 assert p.worker.dead_lettered == 0 and p.worker.failures == 0 assert p.worker.backlog() == 0 report = verify.verify(p.db, settings) @@ -438,6 +466,17 @@ def test_dashboard_reads(client, tokens, replayed): assert report["model"]["test"]["risk_score"] and report["policy"]["tier_counts"] assert report["model_version"] == report["policy"]["model_version"] + case = client.get("/v1/model/business", headers=analyst).json() + assert len(case["points"]) == len(report["insights"]["impact"]) + assert case["assumptions"] == case["defaults"] and case["sensitivity"]["rows"] + bigger = client.post( + "/v1/model/business", headers=analyst, json={"monthly_payments": 40_000_000} + ).json() + assert bigger["scam_loss_at_risk"] == 2 * case["scam_loss_at_risk"] + bad = client.post("/v1/model/business", headers=analyst, json={"abandon_rate": 2}) + assert bad.status_code == 422 + assert client.post("/v1/model/business", headers=analyst, json={"wage": 1}).status_code == 422 + policy = client.get("/v1/policy", headers=analyst).json() assert policy["tiers"]["hold"]["human_review"] is True assert {"en", "bn"} == set(policy["messages"]["scam"]["hold"]) @@ -445,7 +484,7 @@ def test_dashboard_reads(client, tokens, replayed): warn, step_up, hold = (policy["resolved_thresholds"][t] for t in ("warn", "step_up", "hold")) assert 0 < warn <= step_up <= hold < 1 - for path in ("/v1/metrics/daily", "/v1/model/report", "/v1/policy"): + for path in ("/v1/metrics/daily", "/v1/model/report", "/v1/model/business", "/v1/policy"): assert client.get(path).status_code == 401 assert client.get(path, headers=tokens("upay-core")).status_code == 403 @@ -828,8 +867,12 @@ def test_a_freeze_needs_two_people(client, tokens, p, replayed, spare): # --------------------------------------------------------------- customers -def _pending(client, tokens, p, tier: str) -> dict: - """A live payment the policy answers with `tier`, found by asking what-if first.""" +_scored: dict[int, dict] = {} # what `_pending` was answered, by txn_id + + +def _pending(client, tokens, p, tier: str, by_model: bool = False) -> dict: + """A live payment the policy answers with `tier`, found by asking what-if first. + `by_model` skips payments into confirmed-fraud wallets, which a rule holds.""" with p.sessions() as s: candidates = s.execute( select(Transaction) @@ -843,6 +886,8 @@ def _pending(client, tokens, p, tier: str) -> dict: ] service = tokens("upay-core") for sender, receiver, amount, device in candidates: + if by_model and receiver in p.scorer.engine.flagged: + continue for scale in (1.0, 0.5, 2.0, 0.25): body = send(p, sender, receiver, round(amount * scale, 2), device_id=device) probe = client.post("/v1/score/what-if", headers=service, json=body).json() @@ -850,6 +895,7 @@ def _pending(client, tokens, p, tier: str) -> dict: continue result = client.post("/v1/score", headers=service, json=body).json() assert result["decision"]["tier"] == tier # what-if and scoring agree + _scored[body["txn_id"]] = result return body pytest.fail(f"no live payment produced a {tier} decision") @@ -897,6 +943,144 @@ def test_a_step_up_needs_verification_and_the_cooling_off(client, tokens, p, rep assert txn.applied_at > txn.ts and decision.customer_response == "proceed" +# ----------------------------------------------------------------- appeals + + +def _at(iso: str) -> datetime: + return datetime.fromisoformat(iso.replace("Z", "+00:00")) + + +def _appeal(client, service, body: dict, **over) -> object: + answer = {"wallet_id": body["sender_id"], "relation": "family", "reason": "My brother's rent"} + return client.post( + f"/v1/customer/transactions/{body['txn_id']}/appeal", headers=service, json=answer | over + ) + + +def test_a_customer_appeal_releases_a_held_payment_once_a_person_agrees( + client, tokens, p, replayed +): + service, analyst = tokens("upay-core"), tokens("analyst1") + body = _pending(client, tokens, p, "hold", by_model=True) + txn_id, sender = body["txn_id"], body["sender_id"] + assert _scored[txn_id]["decision"]["cue"] != "reported_recipient" + record = client.get(f"/v1/decisions/{txn_id}", headers=analyst).json() + initiated = p.scorer.engine.wallets[sender].n_init + + assert _appeal(client, service, body, wallet_id="W0").status_code == 404 # not theirs + assert _appeal(client, analyst, body).status_code == 403 # only the app files one + assert _appeal(client, service, body, relation="cousin").status_code == 422 + filed = _appeal(client, service, body, reason="Rent for 01711000000, my brother") + assert filed.status_code == 201, filed.text + appeal = filed.json() + assert (appeal["status"], appeal["tier"], appeal["txn_status"]) == ("pending", "hold", "held") + assert "reason" not in appeal and "decided_by" not in appeal # the app sees where, not who + waited = _at(appeal["sla_due_at"]) - _at(appeal["filed_at"]) + assert waited.total_seconds() == 30 * 60 # the hold's own review deadline + again = _appeal(client, service, body) + assert again.status_code == 409 and again.json()["error"]["code"] == "appeal_exists" + status = f"/v1/customer/transactions/{txn_id}/appeal" + assert client.get(status, headers=service, params={"wallet_id": "W0"}).status_code == 404 + assert ( + client.get(status, headers=service, params={"wallet_id": sender}).json()["id"] + == (appeal["id"]) + ) + + assert client.get("/v1/appeals", headers=service).status_code == 403 + queue = client.get("/v1/appeals", headers=analyst).json() + [row] = [a for a in queue["appeals"] if a["id"] == appeal["id"]] + assert row["txn_status"] == "held" and row["relation"] == "family" + assert "01711000000" not in row["reason"] # what a customer typed is masked + assert queue["counts"]["pending"] >= 1 + + path = f"/v1/appeals/{appeal['id']}/approve" + assert client.post(path, headers=analyst, json={"note": "ok"}).status_code == 422 + done = client.post(path, headers=analyst, json={"note": REASON}) + assert done.status_code == 200, done.text + assert done.json()["released"] is True and done.json()["txn_status"] == "completed" + assert done.json()["appeal"]["status"] == "approved" + assert done.json()["appeal"]["decided_by"] is not None + assert p.scorer.engine.wallets[sender].n_init == initiated + 1 + twice = client.post(path, headers=analyst, json={"note": REASON}) + assert twice.status_code == 409 and twice.json()["error"]["code"] == "already_decided" + + with p.sessions() as s: + actions = set( + s.scalars( + select(AuditLog.action).where( + AuditLog.object_type == "appeal", AuditLog.object_id == str(appeal["id"]) + ) + ) + ) + labels = feedback.labels(s).set_index("txn_id") + assert actions == {"appeal.file", "appeal.approve"} + # A person agreed the payment was genuine: it becomes a `legitimate` label. + assert (labels.loc[txn_id, "y"], labels.loc[txn_id, "source"]) == (0, "appeal") + if record["case_id"] is not None: + timeline = client.get(f"/v1/cases/{record['case_id']}", headers=analyst).json() + kinds = [e["kind"] for e in timeline["timeline"]] + assert "appeal_filed" in kinds and "appeal_approved" in kinds + + +def test_an_appeal_never_releases_money_to_confirmed_fraud(client, tokens, p, replayed, spare): + service, analyst, supervisor = tokens("upay-core"), tokens("analyst1"), tokens("supervisor1") + body, result = held_payment(client, tokens, p, spare) + assert result["decision"]["cue"] == "reported_recipient" + appeal = _appeal(client, service, body, relation="friend").json() + case_id = result["decision"]["case_id"] + client.post(f"/v1/cases/{case_id}/escalate", headers=analyst, json={"reason": REASON}) + path = f"/v1/appeals/{appeal['id']}" + refused = client.post(f"{path}/reject", headers=analyst, json={"note": REASON}) + assert refused.status_code == 403 and refused.json()["error"]["code"] == "needs_supervisor" + blocked = client.post(f"{path}/approve", headers=supervisor, json={"note": REASON}) + assert blocked.status_code == 409 + assert blocked.json()["error"]["code"] == "recipient_confirmed_fraud" + rejected = client.post(f"{path}/reject", headers=supervisor, json={"note": REASON}) + assert rejected.status_code == 200, rejected.text + assert (rejected.json()["released"], rejected.json()["txn_status"]) == (False, "held") + with p.sessions() as s: + assert s.get(Transaction, body["txn_id"]).status == "held" # the case still decides + assert body["txn_id"] not in set(feedback.labels(s)["txn_id"]) # no label from a rejection + + +def test_an_appeal_against_a_warning_is_feedback_and_moves_no_money(client, tokens, p, replayed): + staff = tokens("analyst2") + body = _pending(client, tokens, p, "warn") + txn_id = body["txn_id"] + # The warning names the scam it looks like, for the app to explain in words. + assert _scored[txn_id]["decision"]["cue"] in { + "impersonation", "prize", "investment", "wrong_send", "not_you", "generic", + "reported_recipient", + } # fmt: skip + # The demo phone files it as the app would. + filed = client.post( + "/v1/demo/appeal", + headers=staff, + json={"txn_id": txn_id, "relation": "seller", "reason": "I buy from this shop every week"}, + ) + assert filed.status_code == 201, filed.text + appeal = filed.json() + assert appeal["tier"] == "warn" and appeal["txn_status"] == "pending_customer" + waited = _at(appeal["sla_due_at"]) - _at(appeal["filed_at"]) + assert waited.total_seconds() == 24 * 3600 # nothing waits on it + shown = client.get("/v1/demo/appeal", headers=staff, params={"txn_id": txn_id}).json() + assert shown["id"] == appeal["id"] + done = client.post( + f"/v1/appeals/{appeal['id']}/approve", headers=tokens("analyst1"), json={"note": REASON} + ) + assert done.status_code == 200 + # The customer still decides a warning; an appeal never answers for them. + assert (done.json()["released"], done.json()["txn_status"]) == (False, "pending_customer") + + allowed = _pending(client, tokens, p, "allow") + nothing = client.post( + "/v1/demo/appeal", + headers=staff, + json={"txn_id": allowed["txn_id"], "relation": "none", "reason": "just checking"}, + ) + assert nothing.status_code == 409 and nothing.json()["error"]["code"] == "not_appealable" + + def test_a_customer_report_opens_a_case(client, tokens, p, replayed, spare): service = tokens("upay-core") reporter, reported = spare(), spare() @@ -1318,6 +1502,78 @@ def test_events_endpoint_queues_for_the_worker(client, tokens, p, settings, spar assert s.get(Transaction, event["txn_id"]) is not None +def _event(body: dict): + return event_adapter.validate_python(body | {"kind": "txn"}).event() + + +def test_two_workers_holding_one_entry_record_one_decision(p, replica, spare): + """Redelivery to a second worker after the first applied the event: the state + changes once, the second worker finishes the decision, the first one's late + commit finds it done.""" + event = _event(send(p, spare(), spare(), amount=60.0, source="replay")) + results, scored = p.scorer.sequence([event]) + assert results[0].status == "completed" and len(scored) == 1 + seq = p.scorer.seq + again, resumed = replica.sequence([event]) # the entry, delivered to another worker + assert again[0].duplicate and [item.resumed for item in resumed] == [True] + assert replica.seq == seq # caught up, applied nothing new + assert len(replica.finish(resumed)) == 1 + assert p.scorer.finish(scored) == [] # the pending row was claimed already + with p.sessions() as s: + decisions = s.scalars(select(Decision).where(Decision.txn_id == event.txn.txn_id)).all() + assert len(decisions) == 1 + assert s.get(PendingDecision, event.txn.txn_id) is None + # Same features, so the same decision as the worker that applied it would have made. + assert decisions[0].features == pytest.approx( + [float(v) for v in scored[0].features], nan_ok=True + ) + + +def test_a_stopped_workers_entries_are_taken_over_in_order(p, settings, spare): + """A consumer read an entry and died. The next entry waits for it; it is taken + over once idle and both are scored, first one first.""" + stream = settings.events_stream + sender = spare() + first = send(p, sender, spare(), amount=70.0) | {"kind": "txn"} + second = send(p, sender, spare(), amount=80.0) | {"kind": "txn"} + p.redis.xadd(stream, {"data": json.dumps(first)}) + p.redis.xreadgroup(GROUP, "test-dead-worker", {stream: ">"}, count=1) # and never acks + p.redis.xadd(stream, {"data": json.dumps(second)}) + worker = Worker( + p.scorer, p.redis, settings.model_copy(update={"claim_idle_ms": 200}), consumer="test-c" + ) + assert worker.drain() == 2 + assert worker.reclaimed == 1 and p.redis.xlen(stream) == 0 + assert p.redis.xpending(stream, GROUP)["pending"] == 0 + with p.sessions() as s: + a, b = s.get(Transaction, first["txn_id"]), s.get(Transaction, second["txn_id"]) + assert a.created_at <= b.created_at + if a.applied_seq and b.applied_seq: + assert a.applied_seq < b.applied_seq + p.redis.xgroup_delconsumer(stream, GROUP, "test-dead-worker") + + +def test_a_replica_follows_what_another_process_changed(client, tokens, p, replica, spare): + """Flags, freezes and payments made through the API reach a worker's copy of the + state before it scores again.""" + wallet, sender = spare(), spare() + flag(client, tokens, p, wallet) + body = send(p, sender, spare(), amount=45.0) + assert client.post("/v1/score", headers=tokens("upay-core"), json=body).status_code == 200 + catch_up(replica) + assert replica.seq == p.scorer.seq and wallet in replica.engine.flagged + probe = Txn(10**12, p.scorer.engine.last_ts + 60, "SEND_MONEY", sender, "wallet", wallet, + "wallet", 500.0, 10_000.0, "", "app", body["district"]) # fmt: skip + mine, theirs = p.scorer.engine.features(probe), replica.engine.features(probe) + assert np.allclose(mine, theirs, equal_nan=True) + for status, change in (("frozen", p.scorer.frozen.add), ("active", p.scorer.frozen.discard)): + with p.scorer.transaction() as s: # what an approved freeze, then a release, does + s.execute(update(Wallet).where(Wallet.wallet_id == sender).values(status=status)) + change(sender) + catch_up(replica) + assert (sender in replica.frozen) == (status == "frozen") + + def test_the_live_feed_needs_a_token(client): assert client.get("/v1/stream/alerts").status_code == 401 @@ -1400,6 +1656,10 @@ def test_a_message_check_advises_and_keeps_nothing( assert row.object_id == wallet and row.detail["level"] == "high" assert "upay-verify" not in json.dumps(row.detail) + # The flagged message's own words are pointed at, in the script they were written in. + assert found["highlights"] + assert all(SCAM_TEXT[w["start"] : w["end"]] == w["text"] for w in found["highlights"]) + for body in ({"wallet_id": wallet, "text": " "}, {"wallet_id": wallet, "text": "x" * 2001}): assert ( client.post("/v1/customer/message-check", headers=service, json=body).status_code == 422 @@ -1418,6 +1678,45 @@ def test_a_message_check_advises_and_keeps_nothing( assert limited.status_code == 429 +def test_a_payment_to_what_a_flagged_message_named_is_warned( + client, tokens, p, spare, text_model, monkeypatch +): + monkeypatch.setattr(p, "intel", text_model) + service, analyst = tokens("upay-core"), tokens("analyst1") + victim, scammer = spare(), spare() + text = ( + f"Sir ami upay head office theke bolchi. Bhul kore apnar account e 3,450 taka chole " + f"gese, ekhoni {scammer} number e ferot pathan, na hole account block hoye jabe." + ) + checked = client.post( + "/v1/customer/message-check", headers=service, json={"wallet_id": victim, "text": text} + ).json() + assert checked["level"] != "none", checked + + before = client.post( + "/v1/customer/recipient-check", + headers=service, + json={"sender_id": victim, "receiver_id": scammer}, + ).json() + assert before["level"] != "none" and before["reasons"][0]["code"] == "follows_flagged_message" + + body = send(p, victim, scammer, amount=3_450.0) + paid = client.post("/v1/score", headers=service, json=body).json() + assert paid["decision"]["tier"] != "allow", paid + record = client.get(f"/v1/decisions/{body['txn_id']}", headers=analyst).json() + assert "follows_flagged_message" in json.dumps(record) + assert text not in json.dumps(record) # the reason, never the message + + # Someone who checked nothing is not affected. + other = spare() + plain = client.post( + "/v1/customer/recipient-check", + headers=service, + json={"sender_id": other, "receiver_id": scammer}, + ).json() + assert "reasons" not in plain + + def test_a_payment_is_verified_from_the_ledger_for_its_receiver_only(client, tokens, p, spare): service = tokens("upay-core") buyer, seller, curious = spare(), spare(), spare() @@ -1575,3 +1874,15 @@ def test_the_schema_matches_the_models(settings): config = Config(str(BACKEND_DIR / "alembic.ini")) config.attributes["url"] = settings.database_url command.check(config) # raises if a migration is missing + + +def test_consortium_routes_are_reviewer_only_and_optional(client, tokens): + assert client.get("/v1/consortium").status_code == 401 + response = client.get("/v1/consortium", headers=tokens("analyst1")) + # The demo state is built by `python -m fraudlens.consortium.simulate`; without it + # the routes say so and nothing else changes. + assert response.status_code in (200, 404), response.text + if response.status_code == 404: + assert response.json()["error"]["code"] == "consortium_not_built" + else: + assert {"feeds", "guarantees", "audit", "results"} <= response.json().keys() diff --git a/backend/tests/test_security.py b/backend/tests/test_security.py new file mode 100644 index 0000000..12bf0c7 --- /dev/null +++ b/backend/tests/test_security.py @@ -0,0 +1,310 @@ +"""Keyrings and rotation, secrets from files, client addresses behind a proxy, ingest signatures. + +No database or Redis: the endpoint behaviour is in test_ingest.py. +""" + +from __future__ import annotations + +import importlib.util +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from fraudlens.config import BACKEND_DIR, DEV_JWT_SECRET, Settings +from fraudlens.platform import ingest +from fraudlens.platform.audit import WorkflowError +from fraudlens.platform.keys import Keyring, load_keyring, main, rotate_jwt, rotate_partner +from fraudlens.platform.security import ( + TokenError, + check_production, + client_address, + issue_token, + read_token, +) +from fraudlens.secret_sources import SecretFilesSource + +SDK_PATH = BACKEND_DIR.parent / "sdk" / "python" / "fraudlens_partner.py" + + +def load_sdk(): + spec = importlib.util.spec_from_file_location("fraudlens_partner", SDK_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# ---------------------------------------------------------------- keyrings + + +def test_a_rotated_jwt_key_keeps_verifying_until_its_tokens_expire(tmp_path): + path = tmp_path / "jwt.json" + t0 = datetime(2026, 10, 7, 9, tzinfo=UTC) + first = rotate_jwt(path, ttl_minutes=480, now=t0) + second = rotate_jwt(path, ttl_minutes=480, now=t0 + timedelta(hours=1)) + ring = Keyring.from_json(path.read_text()) + later = t0 + timedelta(hours=2) + assert ring.signing(now=later).kid == second.kid + assert ring.verifying(first.kid, now=later) is not None + # 480 minutes after the rotation (and a minute's grace), the old key is done. + gone = t0 + timedelta(hours=1, minutes=482) + assert ring.verifying(first.kid, now=gone) is None + assert ring.verifying(second.kid, now=gone) is not None + # The next rotation prunes it; the file is for the owner only. + rotate_jwt(path, ttl_minutes=480, now=gone) + assert first.kid not in Keyring.from_json(path.read_text()).keys + assert path.stat().st_mode & 0o077 == 0 + + +def test_tokens_carry_the_key_id_and_survive_a_rotation(tmp_path): + path = tmp_path / "jwt.json" + settings = Settings(jwt_keyring=path) + rotate_jwt(path, settings.jwt_ttl_minutes) + old = issue_token(7, "analyst", settings) + rotate_jwt(path, settings.jwt_ttl_minutes) + new = issue_token(7, "analyst", settings) + import jwt + + assert jwt.get_unverified_header(old)["kid"] != jwt.get_unverified_header(new)["kid"] + assert read_token(old, settings)["sub"] == read_token(new, settings)["sub"] == "7" + # Retiring the old key (a leak) refuses its tokens at once, without a restart. + main(["retire", str(path), jwt.get_unverified_header(old)["kid"]]) + with pytest.raises(TokenError): + read_token(old, settings) + assert read_token(new, settings)["sub"] == "7" + + +def test_without_a_keyring_the_single_secret_still_works(): + settings = Settings(jwt_secret="x" * 40) + token = issue_token(1, "admin", settings) + assert read_token(token, settings)["role"] == "admin" + with pytest.raises(TokenError): + read_token(token, Settings(jwt_secret="y" * 40)) + + +def test_partner_rotation_has_a_grace_period(tmp_path): + path = tmp_path / "ingest.json" + t0 = datetime(2026, 10, 7, tzinfo=UTC) + old = rotate_partner(path, "upay", timedelta(hours=24), "https://core.example/cb", now=t0) + new = rotate_partner(path, "upay", timedelta(hours=24), now=t0 + timedelta(days=30)) + ring = load_keyring(path) + assert ring.signing("upay", now=t0 + timedelta(days=30, hours=1)).kid == new.kid + assert ring.verifying(old.kid, now=t0 + timedelta(days=30, hours=23)) is not None + assert ring.verifying(old.kid, now=t0 + timedelta(days=31, hours=1)) is None + assert ring.callback_url("upay") == "https://core.example/cb" # kept across rotations + assert ring.signing(None) is None # partner keys never sign tokens + with pytest.raises(ValueError): + rotate_partner(path, "Not A Name", timedelta(hours=1)) + + +def test_the_production_guard_covers_the_keyrings(tmp_path): + prod = { + "environment": "production", + "database_url": "postgresql+psycopg://fraudlens:s3cret-long@db:5432/fraudlens", + "redis_url": "redis://:pw@redis:6379/0", + "cors_origins": ["https://console.example"], + } + jwt_path = tmp_path / "jwt.json" + rotate_jwt(jwt_path, 480) + check_production(Settings(jwt_keyring=jwt_path, **prod)) # the dev secret is not used + ring = Keyring.from_json(jwt_path.read_text()) + ring.add("jwt") + weak = next(iter(ring.keys.values())) + ring.keys[weak.kid] = type(weak)(weak.kid, DEV_JWT_SECRET[:20], weak.created) + ring.save(jwt_path) + with pytest.raises(RuntimeError, match="32"): + check_production(Settings(jwt_keyring=jwt_path, **prod)) + ingest_path = tmp_path / "ingest.json" + rotate_partner(ingest_path, "upay", timedelta(hours=1), "http://core.example/cb") + with pytest.raises(RuntimeError, match="https"): + check_production(Settings(jwt_secret="k" * 40, ingest_keyring=ingest_path, **prod)) + with pytest.raises(ValueError, match="every address"): + check_production(Settings(trusted_proxies=["0.0.0.0/0"])) + + +# ---------------------------------------------------------------- secrets from files + + +def test_settings_read_secrets_from_files(tmp_path, monkeypatch): + (tmp_path / "jwt").write_text("from-a-file-" + "z" * 30 + "\n") + (tmp_path / "origins").write_text('["https://console.example"]') + monkeypatch.setenv("FRAUDLENS_JWT_SECRET_FILE", str(tmp_path / "jwt")) + monkeypatch.setenv("FRAUDLENS_CORS_ORIGINS_FILE", str(tmp_path / "origins")) + settings = Settings() + assert settings.jwt_secret.get_secret_value() == "from-a-file-" + "z" * 30 # newline gone + assert settings.cors_origins == ["https://console.example"] + monkeypatch.setenv("FRAUDLENS_JWT_SECRET", "both") + with pytest.raises(ValueError, match="not both"): + Settings() + + +def provide(names: list[str]) -> dict[str, str]: + """A secrets provider for the test: knows one secret.""" + return {"seed_password": "from-the-vault-123"} if "seed_password" in names else {} + + +def test_a_secrets_provider_fills_what_is_unset(monkeypatch): + monkeypatch.setenv("FRAUDLENS_SECRETS_PROVIDER", f"{__name__}:provide") + assert Settings().seed_password.get_secret_value() == "from-the-vault-123" + monkeypatch.setenv("FRAUDLENS_SEED_PASSWORD", "the-environment-wins") + assert Settings().seed_password.get_secret_value() == "the-environment-wins" + source = SecretFilesSource(Settings, environ={"FRAUDLENS_SECRETS_PROVIDER": "nonsense"}) + with pytest.raises(ValueError, match="package.module:function"): + source() + + +# ---------------------------------------------------------------- client address + + +@pytest.mark.parametrize( + ("peer", "forwarded", "expected"), + [ + ("203.0.113.9", [], "203.0.113.9"), # no proxy in the way + ("203.0.113.9", ["1.2.3.4"], "203.0.113.9"), # not our proxy: the header is ignored + ("10.0.0.2", ["198.51.100.7"], "198.51.100.7"), # our proxy says who it was + # A client that writes its own header gets its own address, not the one it chose. + ("10.0.0.2", ["6.6.6.6, 198.51.100.7"], "198.51.100.7"), + ("10.0.0.2", ["6.6.6.6", "198.51.100.7, 10.0.0.3"], "198.51.100.7"), # two of ours + ("10.0.0.2", ["garbage, 198.51.100.7"], "198.51.100.7"), + ("10.0.0.2", ["198.51.100.7, garbage"], "10.0.0.2"), # broken hop: stop at ours + ("10.0.0.2", [], "10.0.0.2"), + ("::ffff:10.0.0.2", ["2001:db8::1"], "2001:db8::1"), + ], +) +def test_forwarded_for_is_believed_only_from_trusted_proxies(peer, forwarded, expected): + assert client_address(peer, forwarded, ["10.0.0.0/24"]) == expected + assert client_address(peer, forwarded, []) == peer # nothing trusted: the peer + + +# ---------------------------------------------------------------- signatures + + +def _ring(secret: str = "s" * 64) -> Keyring: + ring = Keyring() + key = ring.add("upay", "upay") + ring.keys[key.kid] = type(key)(key.kid, secret, key.created, "upay") + return ring + + +def test_sdk_and_server_agree_on_the_signature(): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + body = b'{"GrpHdr":{}}' + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", body) + assert ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", body, 300) == key + # And the other way: a callback signed here is accepted by the partner's SDK. + back = ingest.signed_headers(key, "POST", "/cb?x=1", body, "n" * 32) + assert sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body) + assert not sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=2", body) + assert not sdk.verify({key.kid: "other"}, back, "POST", "/cb?x=1", body) + seen = set() + assert sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body, seen_nonce(seen)) + assert not sdk.verify({key.kid: key.secret}, back, "POST", "/cb?x=1", body, seen_nonce(seen)) + + +def seen_nonce(seen: set): + def check(nonce: str) -> bool: + if nonce in seen: + return True + seen.add(nonce) + return False + + return check + + +@pytest.mark.parametrize( + ("change", "code"), + [ + ({"X-FraudLens-Signature": None}, "unsigned"), + ({"X-FraudLens-Key-Id": "nobody-20261007-0000"}, "unknown_key"), + ({"X-FraudLens-Timestamp": "1000"}, "stale_timestamp"), + ({"X-FraudLens-Timestamp": "+1000"}, "stale_timestamp"), + ({"X-FraudLens-Nonce": "short"}, "bad_nonce"), + ({"X-FraudLens-Signature": "v1=" + "0" * 64}, "bad_signature"), + ({"X-FraudLens-Signature": "md5=abc"}, "bad_signature"), + ], +) +def test_bad_requests_are_refused_with_the_check_that_failed(change, code): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + body = b"{}" + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", body) + for name, value in change.items(): + if value is None: + headers.pop(name) + else: + headers[name] = value + with pytest.raises(WorkflowError) as caught: + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", body, 300) + assert (caught.value.status, caught.value.code) == (401, code) + + +def test_the_signature_covers_the_body_path_and_time(): + sdk, ring = load_sdk(), _ring() + key = ring.signing("upay") + headers = sdk.signed_headers(key.kid, key.secret, "POST", "/v1/ingest/transactions", b"{}") + for path, body in [ + ("/v1/ingest/transactions", b'{"a":1}'), # body changed + ("/v1/ingest/transactions?wait=decision", b"{}"), # asks for something else + ("/v1/ingest/transactions/batch", b"{}"), # another endpoint + ]: + with pytest.raises(WorkflowError, match="does not match"): + ingest.verify(ring, headers, "POST", path, body, 300) + then = int(headers["X-FraudLens-Timestamp"]) + with pytest.raises(WorkflowError, match="within 300"): + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", b"{}", 300, then + 301) + ingest.verify(ring, headers, "POST", "/v1/ingest/transactions", b"{}", 300, then - 299) + + +# ---------------------------------------------------------------- pacs.008 + + +def _transfer(**over) -> dict: + sdk = load_sdk() + args = { + "txn_id": 42, + "end_to_end_id": "E2E-42", + "accepted_at": datetime(2026, 10, 7, 9, tzinfo=UTC), + "purpose": "SEND_MONEY", + "amount": 900.0, + "debtor": ("W1", "wallet"), + "creditor": ("W2", "wallet"), + "channel": "app", + "district": "Dhaka", + "debtor_balance_before": 20_000.0, + "ip": "203.0.113.9", + } + return sdk.credit_transfer(**(args | over)) + + +def test_a_pacs008_message_maps_onto_the_platform_event(): + sdk = load_sdk() + body = json.dumps(sdk.pacs008("MSG-1", [_transfer()])).encode() + message, [txn] = ingest.parse(body, single=True) + assert message.GrpHdr.MsgId == "MSG-1" + assert (txn.txn_id, txn.type, txn.sender_id, txn.receiver_id) == (42, "SEND_MONEY", "W1", "W2") + assert (txn.amount, txn.sender_balance_before, txn.channel) == (900.0, 20_000.0, "app") + assert txn.event().txn.network == "203.0.113.0/24" # the address itself is not kept + + +def test_pacs008_problems_are_reported_where_the_partner_put_them(): + sdk = load_sdk() + wrong = _transfer(purpose="CASH_OUT", debtor_balance_before=None) # a wallet to a wallet + body = json.dumps(sdk.pacs008("MSG-2", [_transfer(), wrong])).encode() + with pytest.raises(ingest.IngestInvalid) as caught: + ingest.parse(body, single=False) + assert all(e["loc"][:2] == ("CdtTrfTxInf", 1) for e in caught.value.errors) + with pytest.raises(ingest.IngestInvalid) as caught: + ingest.parse(body, single=True) + assert caught.value.errors[0]["loc"] == ("CdtTrfTxInf",) + unknown = json.loads(body) + unknown["CdtTrfTxInf"][0]["Surprise"] = 1 + with pytest.raises(ValueError): + ingest.parse(json.dumps(unknown).encode(), single=False) + + +def test_the_example_and_the_sdk_have_no_third_party_imports(): + for path in SDK_PATH.parent.glob("*.py"): + text = Path(path).read_text() + assert "import httpx" not in text and "import requests" not in text diff --git a/backend/tests/test_simulator.py b/backend/tests/test_simulator.py index bba6348..9c16a4e 100644 --- a/backend/tests/test_simulator.py +++ b/backend/tests/test_simulator.py @@ -1,4 +1,7 @@ +from dataclasses import fields + import pandas as pd +import pytest from fraudlens.simulator.config import SimConfig from fraudlens.simulator.engine import Simulation @@ -90,3 +93,37 @@ def test_same_seed_gives_identical_data(small_tables): sim.run() again = build_tables(sim)["transactions"] pd.testing.assert_frame_equal(again, small_tables["transactions"]) + + +def test_default_profile_is_the_published_world(): + assert SimConfig().with_profile("default") == SimConfig() + assert SimConfig.small().with_profile("default") == SimConfig.small() + + +def test_calibrated_profile_changes_only_sourced_parameters(): + base, cal = SimConfig.small(), SimConfig.small().with_profile("calibrated") + changed = {f.name for f in fields(SimConfig) if getattr(base, f.name) != getattr(cal, f.name)} + assert changed == { + "profile", + "txn_cap", + "send_median", + "cash_out_median", + "report_rate", + "ato_keep_share", + } + + +def test_unknown_profile_is_rejected(): + with pytest.raises(ValueError): + SimConfig().with_profile("real") + + +def test_calibrated_world_is_capped_and_has_fewer_takeovers(small_tables): + cfg = SimConfig.small().with_profile("calibrated") + sim = Simulation(cfg) + sim.run() + cal = build_tables(sim) + t = cal["transactions"] + assert t.loc[t["sender_type"] == "wallet", "amount"].max() <= cfg.txn_cap + ato = (cal["cases"]["typology"] == "account_takeover").mean() + assert ato < (small_tables["cases"]["typology"] == "account_takeover").mean() diff --git a/backend/tests/test_survey_analysis.py b/backend/tests/test_survey_analysis.py new file mode 100644 index 0000000..ca80ed1 --- /dev/null +++ b/backend/tests/test_survey_analysis.py @@ -0,0 +1,64 @@ +"""The survey fixture is invented: every respondent_id starts with FAKE-.""" + +from pathlib import Path + +import pandas as pd +import pytest + +from fraudlens.validation.survey_analysis import SurveyError, load, summarise, summarise_by + +FIXTURE = Path(__file__).parent / "fixtures" / "survey_fake.csv" + + +@pytest.fixture +def survey(): + return load(FIXTURE) + + +def test_fixture_is_obviously_fake(survey): + assert survey["respondent_id"].str.startswith("FAKE-").all() + + +def test_rows_without_consent_are_ignored(survey): + s = summarise(survey) + assert s["responses"] == 5 and s["consented"] == 4 + assert s["q04_loss_bdt"]["n"] == 2 # FAKE-005's Tk 99,999 is not counted + assert s["q04_loss_bdt"]["median"] == 3000.0 + + +def test_shares_and_victim_only_questions(survey): + s = summarise(survey) + assert s["q01_contacted"] == {"n": 4, "yes": 0.75, "no": 0.25} + assert s["q02_lost_money"]["yes"] == 0.5 + assert s["q03_scam_type"]["n"] == 2 + assert s["q03_scam_type"]["impersonation"] == 0.5 + assert s["q05_reported_to"]["nobody"] == 0.5 + + +def test_screen_reactions(survey): + screens = summarise(survey)["screens"] + assert screens["warn"] == {"n": 4, "mean": 4.0, "agree": 0.75, "disagree": 0.25} + assert screens["hold"]["agree"] == 0.25 + assert screens["step_up"]["mean"] == 4.25 + + +def test_split_by_area(survey): + out = summarise_by(survey, "area") + assert set(out["groups"]) == {"rural", "urban"} + assert out["groups"]["urban"]["consented"] == 2 + + +def test_bad_answers_are_rejected(tmp_path): + bad = pd.read_csv(FIXTURE, dtype=str, keep_default_na=False) + bad.loc[0, "q07_warn_would_stop"] = "7" + path = tmp_path / "bad.csv" + bad.to_csv(path, index=False) + with pytest.raises(SurveyError): + summarise(load(path)) + + +def test_missing_required_column(tmp_path): + path = tmp_path / "no_consent.csv" + path.write_text("respondent_id,area\nFAKE-001,urban\n") + with pytest.raises(SurveyError): + load(path) diff --git a/backend/uv.lock b/backend/uv.lock index 3873d21..0aba1dd 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -249,6 +249,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/11/1293429c1c3a3e19b551275e65efddd122a905bbe7e368816a59f3ef2a41/cuda_bindings-13.4.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bfbd3f7d4ac04dd41dc49121b9e408c8283992f47124c2290ecb79bbbadcca8e", size = 6488226, upload-time = "2026-09-23T02:22:00.578Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c3/efb6bbb7307bf5c83dc4acca650280b210c67ed1a1a60f898a90e7c82e38/cuda_bindings-13.4.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5f72bcfcdf3be23e1da3c792f68f508586f48d037bca8b10f552c4cca5971f2", size = 7172185, upload-time = "2026-09-23T02:22:03.114Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a9/c83eb5aa055a4b0c3776d83f6f88b9e778a6fe0415210977c889c6a0bb8a/cuda_bindings-13.4.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7c6c9f46fca7f3fc61959ef9a2398ac656172145b43f408e0a6492360cf1c0c", size = 6316533, upload-time = "2026-09-23T02:22:09.694Z" }, + { url = "https://files.pythonhosted.org/packages/8a/24/9c01edfd2210737ee9471b47db857a079e5a23f2677e5d9778c0ff23d099/cuda_bindings-13.4.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fd7d8459b364aedc11f3e59703453ced823135f78a9111ca70feef8d56d4d21", size = 6929124, upload-time = "2026-09-23T02:22:11.765Z" }, + { url = "https://files.pythonhosted.org/packages/ab/e6/3c094ef0eb00a7b0ff69a3915327e2c2d14e712ebe471a2217bf7f020f33/cuda_bindings-13.4.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4796864ce829bd95ef2ef0d23c6ba21bb64e08f7fab0a377302ed1affb6605c7", size = 6407948, upload-time = "2026-09-23T02:22:18.484Z" }, + { url = "https://files.pythonhosted.org/packages/a3/49/7a3769c43e432b0434dd46424058b47af4347167f0dfca1ecb27e2de92a1/cuda_bindings-13.4.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbacde6f75665b197016b986164cfdaa33b17515e5e635a63ddb75926aaa71c3", size = 6978955, upload-time = "2026-09-23T02:22:20.535Z" }, + { url = "https://files.pythonhosted.org/packages/0a/ca/2c4419ca787278f65faf0f0155791a80fa141f39a628e97e4663e2ba09fa/cuda_bindings-13.4.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6eb969920e28f66f8fc3b0b3afcb6e09381cc96bf8e8158d774e9488ae89980", size = 6314272, upload-time = "2026-09-23T02:22:26.785Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f878de5de8e6d1a64d55096539b7b72821e6dc62682d5968e842b95d97df/cuda_bindings-13.4.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e11cfe8fec4c85ce79feda18124971c52596f0cbd642a94f5dafc257124a4b3", size = 6865978, upload-time = "2026-09-23T02:22:29.041Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.8.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/fb/f8e1890428f9f590b4beebd63b068aac1ce32a3331510c847b9f9a78f261/cuda_pathfinder-1.8.3-py3-none-any.whl", hash = "sha256:e29e59829c297a7a5233bd9cc71094fc5bddbd076951482670178f9eade39b1f", size = 62561, upload-time = "2026-10-02T03:20:23.712Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +curand = [ + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusolver = [ + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] + [[package]] name = "docstring-parser" version = "0.18.0" @@ -275,6 +350,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/b6/78aaf9141fb46742928c113f3cf6ef2259d538cb02b604b7656c1dc9883c/fastapi-0.142.2-py3-none-any.whl", hash = "sha256:bd5f4d81f1e93a88bcd77caf4dfe3c2dbffc3805407a0007e9a114c18b3a670b", size = 144409, upload-time = "2026-09-30T09:51:09.8Z" }, ] +[[package]] +name = "filelock" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/e4/34efcb869715cf299e47d1ac7b2624d2bcb6f2d3dffc2f0abe8417f65ab2/filelock-4.0.12.tar.gz", hash = "sha256:cf42711a7ac791818b299fab0332a088c65aeeefa36290de98db92c434303b0c", size = 253880, upload-time = "2026-10-05T13:17:43.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/a1/b047e1c486a2c66283633b7dbb73872ca16ca1ce6a426f70d4d037fc8e1c/filelock-4.0.12-py3-none-any.whl", hash = "sha256:5f17ee83ecee8a6f3e389c75822fb70a1c2f0506b99438a6dffa1d56793588c8", size = 111732, upload-time = "2026-10-05T13:17:42.361Z" }, +] + [[package]] name = "fraudlens" version = "0.1.0" @@ -300,6 +384,11 @@ dependencies = [ { name = "uvicorn", extra = ["standard"] }, ] +[package.optional-dependencies] +transformer = [ + { name = "sentence-transformers" }, +] + [package.dev-dependencies] dev = [ { name = "pytest" }, @@ -325,9 +414,11 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3" }, { name = "redis", specifier = ">=8.1.0" }, { name = "scikit-learn", specifier = ">=1.9.1" }, + { name = "sentence-transformers", marker = "extra == 'transformer'", specifier = ">=3.0" }, { name = "sqlalchemy", specifier = ">=2.1.1" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.54.0" }, ] +provides-extras = ["transformer"] [package.metadata.requires-dev] dev = [ @@ -336,6 +427,15 @@ dev = [ { name = "ruff", specifier = ">=0.16.9" }, ] +[[package]] +name = "fsspec" +version = "2026.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz", hash = "sha256:0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", size = 333545, upload-time = "2026-09-18T17:50:42.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl", hash = "sha256:8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", size = 221738, upload-time = "2026-09-18T17:50:41.341Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -345,6 +445,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "hf-xet" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/27/06d899ea7bd721d272f84aac98bdb238de98af4cc767a69056d967d68c71/hf_xet-1.7.0.tar.gz", hash = "sha256:d406ec79053c0871817f700c2ac8c36ba0d87f9c34b7458b0f0063bb218b0466", size = 985689, upload-time = "2026-10-06T20:18:43.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/7c/3e45174942e6793adde6cba4daa7fb037275cf02a944d9eadfcf9ff33b86/hf_xet-1.7.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:fa029678be1ba7f953c409b0b27bf15cc69cd1c9b3a674fbd78856ebefca1052", size = 3803919, upload-time = "2026-10-06T20:18:09.844Z" }, + { url = "https://files.pythonhosted.org/packages/ff/3a/5e8b363391adcbb002e191dbf924dab31464ea9c45adfeb73502afc36d35/hf_xet-1.7.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:57bc157b8b7fe3bee9dcb9af7f3da8de41801c3b31a9ef68a77a33c6a6be382f", size = 3553588, upload-time = "2026-10-06T20:18:13.376Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c2/0d1eaa5da13bbf9c896badc7f380601c7d973a87a6ffb4d100267c4536c1/hf_xet-1.7.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:87dab080f8f7d32781c2586904e3603f4e60d09bfc727706c3ae419e0829beeb", size = 4201962, upload-time = "2026-10-06T20:18:16.11Z" }, + { url = "https://files.pythonhosted.org/packages/23/2d/225d5b11a9ca7d31b9470a57f2b2be1a5cef8b84325a2146aeb4589e226c/hf_xet-1.7.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b01fe18dbbd151a2403d2c64ed30dc6547b00d6babab9a617d77c7acdb81ee66", size = 3982978, upload-time = "2026-10-06T20:18:18.092Z" }, + { url = "https://files.pythonhosted.org/packages/93/34/9d681f0e3dac0b5dae0d7dea748429266f24e52415446523f464fbaa828e/hf_xet-1.7.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4ee5e05a627f5ab5bad7a86582277d645556ea1e199903aae19e033a392aa13a", size = 4181558, upload-time = "2026-10-06T20:18:20.082Z" }, + { url = "https://files.pythonhosted.org/packages/de/f0/277f039b7d72027bc2ed277f1b62a2f70f740a5aac2a3e7243e5b6854c5d/hf_xet-1.7.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19c0e64f14175ccb6a1aff69e0d2ab9ec5269a560e6687abaf2b3fa4f73de7cd", size = 4411546, upload-time = "2026-10-06T20:18:21.999Z" }, + { url = "https://files.pythonhosted.org/packages/3d/7f/832d3ddb49326114175b7bcc50daea8565c09fd21ac03a02b211c09fefb7/hf_xet-1.7.0-cp314-cp314t-win_amd64.whl", hash = "sha256:757168feb5679647c0bb13ee5d0faebe799c4dff9051419885a566ebd79f949d", size = 3812809, upload-time = "2026-10-06T20:18:24.288Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c4/310c3c29e5beae7c049e63947bd1923d597883b41c9ec4718589920812c4/hf_xet-1.7.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b91569d5f1b61c34b043687da02c05dd3604f3d329e7868510bf3f7971599006", size = 3646174, upload-time = "2026-10-06T20:18:26.279Z" }, + { url = "https://files.pythonhosted.org/packages/9c/0b/b03be21ffaada749ba0d3197d8aefbf1aa698bac149580421c15239b299e/hf_xet-1.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e3e88a7a75d7d95cbee1f37dc31341d6201124cf21c6c4b1dfab8ccba9b09e0f", size = 3796096, upload-time = "2026-10-06T20:18:28.43Z" }, + { url = "https://files.pythonhosted.org/packages/c3/47/a26ebdce7056a61e931f228439bc0ab08cbec239d1690f965e5e637cba79/hf_xet-1.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:59fba37039233c7fcbe196817d6cdcf1b40dfb17b410f229d85b0cf0a1848da4", size = 3560352, upload-time = "2026-10-06T20:18:30.365Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4c/2bf3b66c215d409655f28de1622393dde04c9461280d48c7924bb3b2decd/hf_xet-1.7.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2814a6e999d13464c4d679b788cc5d784eb5a4edfc638a31f10e9a11ab531ef8", size = 4212180, upload-time = "2026-10-06T20:18:32.292Z" }, + { url = "https://files.pythonhosted.org/packages/49/0c/a2f703a5a78267556e89e03316fa0805c86b72b50829bc67665746e8ebf0/hf_xet-1.7.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fcfd6c22418e57dd5b3aea649e813b2e2cfb2aebf317b210d90f1fe4b3018b52", size = 3990011, upload-time = "2026-10-06T20:18:34.21Z" }, + { url = "https://files.pythonhosted.org/packages/a4/77/e52e4201b1cbf571530a61cc57f70182045a39a230089ee5f1df182a4de2/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80f79dae613ce9e0ea1fd1ae15616ca9ac74aed4c770aabc199c4f03ebecc863", size = 4190628, upload-time = "2026-10-06T20:18:36.062Z" }, + { url = "https://files.pythonhosted.org/packages/6c/dc/03a21b89f118664a0926ff25b0f8e44a519bf22724a6a8fc7a9abbc188b6/hf_xet-1.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:0a9e802f33bf50c851abe45fc5380e61f959e2d369647d6742b79ad9d6c27cab", size = 4418814, upload-time = "2026-10-06T20:18:37.888Z" }, + { url = "https://files.pythonhosted.org/packages/4d/59/b35106dfa71b6eef605dc88bd038fe99c7f86fb132a15b60d0bf2f235b2c/hf_xet-1.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:2b7bb5727889b0f2436dbaaad8fc4c3e66b8240d992716989e0c086b4278b1bc", size = 3822644, upload-time = "2026-10-06T20:18:40.052Z" }, + { url = "https://files.pythonhosted.org/packages/48/cd/072313585f74fe9d441e2eb5e0a4703c30586cd709810ea369675f61b74e/hf_xet-1.7.0-cp38-abi3-win_arm64.whl", hash = "sha256:acc3851cf2576a8fb2ae926da863f4efabe21303cf292e9a44332802ab0dcc6a", size = 3662436, upload-time = "2026-10-06T20:18:42.205Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -448,6 +572,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, ] +[[package]] +name = "huggingface-hub" +version = "1.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/2a/484d112c0d8fc5f665d7b65137ac9cdb2953c982391598c3597968a12ee7/huggingface_hub-1.33.0.tar.gz", hash = "sha256:367be21a201db9523eddf8aeac7048f2602c1b308691c97640d5e72ed188007e", size = 1043548, upload-time = "2026-09-24T09:49:29.971Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/16/963096d224b80909432dc16561a615fd33d2d13beef3ce4c63fa25e40867/huggingface_hub-1.33.0-py3-none-any.whl", hash = "sha256:04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899", size = 846435, upload-time = "2026-09-24T09:49:28.059Z" }, +] + [[package]] name = "idna" version = "3.20" @@ -466,6 +610,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.17.0" @@ -603,6 +759,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -666,6 +834,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + [[package]] name = "narwhals" version = "2.26.0" @@ -757,6 +943,158 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/0b/b12a2df5d1b774bd9007a6fdff9381145b6223d37f11afc9c37ab0efd9a1/numpy-2.5.3-cp315-cp315t-win_arm64.whl", hash = "sha256:befa1ae5bd6030b3f512b43ff3fa5290bbed6b84411a44244b14adf835f5b89d", size = 10850807, upload-time = "2026-09-06T16:27:43.868Z" }, ] +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.24.0.43" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ba/791cffd048fe5b044e620df55267e3e95c0e6e07d50b41e377c03dfc910f/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:71f181cd810e90f9b6023b01186fe82d13d65f0ec098581ee201d39fad769e4b", size = 553099438, upload-time = "2026-07-02T16:27:42.58Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.30.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/21/a73174c6157101bdf1ffc22b517f76ff0082613989dd9bc8f43e8034caac/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:ca786ffa5a647c75d4d1f5cc72a6c4f537947e2ba8823d7c8aaf768e7a7b9f77", size = 215983881, upload-time = "2026-06-09T03:23:15.633Z" }, + { url = "https://files.pythonhosted.org/packages/3f/34/c500f90c7ae641b8e0f98965b36b8a7ac79cc8b296e8d251fe3eb592ee54/nvidia_nccl_cu13-2.30.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:cefa7fdb9710efd0f39c5f1be1d61ff6fc9a996c451265bd7fbdcf9455ed4b50", size = 215965170, upload-time = "2026-06-09T03:23:39.73Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.4.92" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1d/6b/eef7a9e32872b8f41e145bf10cddc9af26e153c338852811fe9a9baddf9e/nvidia_nvjitlink-13.4.92-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:e0391f24ed94ec879b84e3da4d4ec320c879aff681f2c7a638462f7199284323", size = 42452378, upload-time = "2026-09-16T20:45:29.042Z" }, + { url = "https://files.pythonhosted.org/packages/1f/a8/1cbd4014898af8b419e69b0d7dbc63da2121ee92d92b47d59f4fe9075349/nvidia_nvjitlink-13.4.92-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:25f74fad0d654271c921ac4dca614bd6258bc21791242fc7b2289dad7ae9c099", size = 40420120, upload-time = "2026-09-16T20:45:19.163Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.45.0" @@ -1190,6 +1528,123 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, ] +[[package]] +name = "regex" +version = "2026.9.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f2/af1da9d3ceed77bfcdce40427d49ba0be94e4fe84245e3bfef68c10e75b6/regex-2026.9.29.tar.gz", hash = "sha256:8b5fcc4771732191b2b7d1dd68d8f0353f47f8d90b6150f6dce58bf1112442cb", size = 419199, upload-time = "2026-09-29T00:49:58.298Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/48/3fdcde9a0baa84d7d25571223265d6e434e114763b438601d54a8028bf3e/regex-2026.9.29-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:dc79d36d0618752265f0d575915bdc5c5130ecb9c9f6b3bcefeae32e4bdfafcf", size = 497903, upload-time = "2026-09-29T00:46:38.938Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1c/4ee3e97c76f53940488dfe7a7e18705e78daac8cd7fb161d246b9e328449/regex-2026.9.29-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3a21a9509d0ee88e7a70e1ad228cd2f0e0fd1e187458db132e8a8d18c97daf9d", size = 296416, upload-time = "2026-09-29T00:46:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/37/14/f3f0ba083d2094392d5eabf56db5ea6ba469fd6e927afd187042054ea68a/regex-2026.9.29-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f57dc6b8fef170f105d2cf5cdce254f47b137d7755086cf7050f47e16582abba", size = 293633, upload-time = "2026-09-29T00:46:41.959Z" }, + { url = "https://files.pythonhosted.org/packages/c9/72/67e7a8ce17f1aea49df215564048efb49cc8c2b31a0e0fc30f36838f8516/regex-2026.9.29-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f93bc1c3486ef3747e07c9d7c1d0a147b8fbaab975f80e348aed6f71309dfaca", size = 805885, upload-time = "2026-09-29T00:46:43.373Z" }, + { url = "https://files.pythonhosted.org/packages/f6/78/25436bcfd4d2260b4b4090094d55d7ab53ec8a1ab4865a0b8bcb33c7d5c0/regex-2026.9.29-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e1d3a4cb7993b708f0ada8d0c84590efd853f169e7147d2202c9da503180242", size = 878344, upload-time = "2026-09-29T00:46:45.328Z" }, + { url = "https://files.pythonhosted.org/packages/97/e6/a09ec3a23ae41d6179880e67f0aace9284b2d95f2d7b326eff203f8eec5e/regex-2026.9.29-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dabee8f4935e731fb46b2a3091bdda0d3d94b3bbfb907d2b4f12eefce4009619", size = 919181, upload-time = "2026-09-29T00:46:47.041Z" }, + { url = "https://files.pythonhosted.org/packages/26/83/d2fbd2e4e3afb1167daa825187d196f313cbaa1a4768f311fb041bb0e3d2/regex-2026.9.29-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39ab5894d971f9ac68baa6eca5c50387db579cfcacf36ae8df3feceb1815e6d0", size = 807783, upload-time = "2026-09-29T00:46:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/46/0b/eb429a7016610d44fc89a597163f8c9127505f0d7dc724dc9effbb6a3ac0/regex-2026.9.29-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c1a9a6651197fbed6f0212591418b9def774fc3f8324f78d1bf0e6a63e5f8aa1", size = 783465, upload-time = "2026-09-29T00:46:50.64Z" }, + { url = "https://files.pythonhosted.org/packages/1b/07/58a3c0153c7476898430f6a7cf3d9062a1d17fbea4f43399ecaf411c7b4c/regex-2026.9.29-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87fb80cbe3557e27e7b28b995c2b2eedf689b8886f941ab93e0e288f0976518a", size = 793519, upload-time = "2026-09-29T00:46:52.396Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/161b94d39164520e21a7befe0245569bf7fda4c7cf1fc4e2df2b5def49da/regex-2026.9.29-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c5c2ef13797466aa64170cbb66ad98a32351dd4127694cea7199f80f213750d", size = 869293, upload-time = "2026-09-29T00:46:54.128Z" }, + { url = "https://files.pythonhosted.org/packages/8f/07/3b02ed829aa2decdc1955d222bd1e2f99d1c8bb4873bbb9a66b2f0a36bff/regex-2026.9.29-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:59b49507f47479e299a9e1bc41b5cb83a7afda0540625f1dbae886615978acbf", size = 770239, upload-time = "2026-09-29T00:46:56.106Z" }, + { url = "https://files.pythonhosted.org/packages/42/5b/ba61f6fe062eb8562e742367d177bb75370434138ef6c9d2a27114f8d613/regex-2026.9.29-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0dd8af32e9f7b56b7f95cc1fd79b23054c3bdc172392ae560acc24d57b7ffe71", size = 861973, upload-time = "2026-09-29T00:46:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/cc/27/767259b20e8a842948990f5e99138d6c077248fd42f8b5468b1d9ca4b814/regex-2026.9.29-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db5e82ba15c142425b8406690032df89e39cca4a2e8afbbb9a3d84edc2373ac3", size = 796470, upload-time = "2026-09-29T00:46:59.236Z" }, + { url = "https://files.pythonhosted.org/packages/a0/05/2566c4ba849b68a8ab81a6bf428fa79d20aae7ddee83979103c0381df254/regex-2026.9.29-cp312-cp312-win32.whl", hash = "sha256:d0c3082bf79bcd6a614d55916590ad4b8f93200e10b97f463ea5d9d07c9b5f23", size = 269327, upload-time = "2026-09-29T00:47:01.135Z" }, + { url = "https://files.pythonhosted.org/packages/93/19/489bc8db91196381c935752df01ba3f607140daece33b78d88573f028e64/regex-2026.9.29-cp312-cp312-win_amd64.whl", hash = "sha256:fdd88ed5e20b1bcdd234421e454962c971aa44b653bdb7f1ea9ef683e90fb649", size = 280334, upload-time = "2026-09-29T00:47:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/47/fb88ba779d0e5e7d4b0ec1aceeb13845948a2cb876bd572a2d1dfdba090b/regex-2026.9.29-cp312-cp312-win_arm64.whl", hash = "sha256:4fe97894d1b306c919b4e50def1e6f6c522f4d03a7283811f4d108f1ce5d3ac2", size = 279614, upload-time = "2026-09-29T00:47:06.541Z" }, + { url = "https://files.pythonhosted.org/packages/79/d5/6080f7d1a6e7e36aa720f806ac93c035ba39c209ae6cc510e8ef4c0279c6/regex-2026.9.29-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f1a0d5117230dd46b399a30a38afa44f79c99f3168988fdc4f425c3f928b39df", size = 497622, upload-time = "2026-09-29T00:47:08.251Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/c87fc7a2e21a42f9d57489db32951c37eef56d153840459a80d464f0321d/regex-2026.9.29-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f0fe9834e5aeccaf19a0d8feb296d66a24be1a7c9922002f842a682cd5abb787", size = 296281, upload-time = "2026-09-29T00:47:09.764Z" }, + { url = "https://files.pythonhosted.org/packages/11/9e/aa0f4cde3bc4688c1d58b0cd8415edd708339bc0bc401a195b0b1e8c8f0c/regex-2026.9.29-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c90fcf7804ea0a54b896ce0f2b9565350220b8d4890fd0db461a476a4c687963", size = 293472, upload-time = "2026-09-29T00:47:11.723Z" }, + { url = "https://files.pythonhosted.org/packages/90/d4/e835c487850ed922a8d6074f953b888c8ea99775c76b9ed5f8a4d72eab92/regex-2026.9.29-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e11edba5bc344a32b029a7af9d4b3173982dd79eeafa0b9dbd787364414b0509", size = 805966, upload-time = "2026-09-29T00:47:13.235Z" }, + { url = "https://files.pythonhosted.org/packages/2c/57/ba8809847fbae8d2cbc71367c6ded510a7ec88bf52493c65efc1acf4effb/regex-2026.9.29-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb90e7177944b6684738c1fc36aabd2dd00d1de3be7dbe09f91e196f1bc0dc81", size = 878346, upload-time = "2026-09-29T00:47:14.877Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/e3da19fc3cc15ef67ab67e121e87887c3bccfdb683a7a9ec557c460ca5b7/regex-2026.9.29-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d06fcdecc10fc7954d7c8f27a03c96055fe525274dc84a7b0dbdc3d6b9e03dab", size = 919250, upload-time = "2026-09-29T00:47:16.622Z" }, + { url = "https://files.pythonhosted.org/packages/9a/8e/c1ed81f55f992f6aa0b699a592a50c1ce9e6d44ff1aee2c14c0537dcef9c/regex-2026.9.29-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d49c18f1ea294cf4adde2e5ac256e98c82ea9d708462ce4bf799dffa7cfe8a2c", size = 807902, upload-time = "2026-09-29T00:47:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/ad/bc/5a6886eb470e41040e21e05b75024a18b6ebfe7ea400b72094a60f949101/regex-2026.9.29-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3e778bfccd63075167709136afbc251c1f683758d5bf49c803c60ac3f894ce6b", size = 783514, upload-time = "2026-09-29T00:47:19.921Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/6d951d453b023c6edb880f1ba474291b53b8ce1cc438b96a9db6d791d991/regex-2026.9.29-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:686ac5350fceae63830bb98805fcb8039325bf4c06d9f6f048ff65229d5bffa5", size = 793466, upload-time = "2026-09-29T00:47:21.552Z" }, + { url = "https://files.pythonhosted.org/packages/99/b9/d5a41adc08360f5eee0dc4846c578f002366947211fc8af5a69a64ee7b9f/regex-2026.9.29-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:26ec4ccce55aa533fbd603d08911b01101a8fcfec987845ac3ae2c7087b2bde3", size = 869464, upload-time = "2026-09-29T00:47:23.276Z" }, + { url = "https://files.pythonhosted.org/packages/4b/32/d76c9d91f5d798e2e9e67f6f85ec4ae35445ac425f7454797311cecb80ca/regex-2026.9.29-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:a655d34b2a6943af32401f3d94f72e9d731f6ad16285815550bf2b4ee69d420a", size = 770278, upload-time = "2026-09-29T00:47:25.193Z" }, + { url = "https://files.pythonhosted.org/packages/24/00/aeebdb540c620a0f7317f6d6fad80a47729ecf0599a24b5c34ec155351f5/regex-2026.9.29-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0c992c19cd45058a4b92f68f139c93db168b48fb1f322c9a7cd620806afb6b51", size = 861949, upload-time = "2026-09-29T00:47:27.005Z" }, + { url = "https://files.pythonhosted.org/packages/12/62/d0314bcedfd3586197e4596931fa220260eb2385bf53184e5b9ae67db24b/regex-2026.9.29-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ebb8912f565b8cdbbf27debfe00df04202c20e2f651b9e32767930c5eace3621", size = 796602, upload-time = "2026-09-29T00:47:29.233Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c7/d5a8c13a613facb03e0fb55c1ebaaf7bb35d8e2c1abe8bef8dca809fc1d9/regex-2026.9.29-cp313-cp313-win32.whl", hash = "sha256:4d7d93613b01b0199961330e49cfc52d479b3d5776c56c691db31130c0a07d91", size = 269306, upload-time = "2026-09-29T00:47:31.14Z" }, + { url = "https://files.pythonhosted.org/packages/80/a7/bf93a3a6afa5f7bc16b7afb94ae581b01cae620b8ad56bd8f9572a985959/regex-2026.9.29-cp313-cp313-win_amd64.whl", hash = "sha256:61956f074ecd123f55adca68ee3eab46e6a07ad3f8e64e6db95dfacb444f55c4", size = 280307, upload-time = "2026-09-29T00:47:32.709Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7d/388274e53605a86297f433a08102a7bbdcf9379d47683d307ccaefd88e2c/regex-2026.9.29-cp313-cp313-win_arm64.whl", hash = "sha256:bfc71e6d970419c1309b3640305298643e2a734cad3f7cfb6d2ddee4175ab53d", size = 279599, upload-time = "2026-09-29T00:47:34.674Z" }, + { url = "https://files.pythonhosted.org/packages/93/1f/d9dc6f02f569625faf67a4daec926cd5023472dcd69bb44286dccd5a5ab3/regex-2026.9.29-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:957bb708e8057ab1649ba566456429d691ec9b90d1c9ad1af1ba7ffbbeaf05f2", size = 497598, upload-time = "2026-09-29T00:47:36.541Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/9b693a3fd1451381e812031a8961ec5b3b8f0c8cc6871f14c5223642804d/regex-2026.9.29-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c9b602fae1e00b7c035d661ce85575365719192a7b46784bd71cf64c68053aa0", size = 296250, upload-time = "2026-09-29T00:47:38.233Z" }, + { url = "https://files.pythonhosted.org/packages/dd/5f/52bc2abc3fef040cd9de76ab29c918d6a717a454ae2b9dd7938b0c95656d/regex-2026.9.29-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0166844493626c5015c6088ee15c9ca2fd060ca15b7641d1657da6a58432ae33", size = 293518, upload-time = "2026-09-29T00:47:39.957Z" }, + { url = "https://files.pythonhosted.org/packages/dc/fc/cf50671215ee0057046980b4571ef8646a005819bb67f0957e779ed107a5/regex-2026.9.29-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b97a38fb4c732b6832db6bf108963adbcd82ef1268ba2025dce390f45af75efa", size = 806037, upload-time = "2026-09-29T00:47:41.676Z" }, + { url = "https://files.pythonhosted.org/packages/14/4b/dddef8fc15c63e4347cc9efb138d0cd306f30e6c98acbcc81a8f780083b9/regex-2026.9.29-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a540abfab208e1b7ef2df231c40ef3b6cbb30a0aad6204e9b6a81c10a6794628", size = 878881, upload-time = "2026-09-29T00:47:43.755Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/38daabed32d28f7e58a06e9344ce00dc67952e9996bc578ed6a29fe1240e/regex-2026.9.29-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ddfa987262763c3c22a8367d2a49c244b018a74c3a8e3ab1a864119ad45c5633", size = 918684, upload-time = "2026-09-29T00:47:45.594Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4d/041d9458a645fee4fce4d642a89d27271a3cfcd91095104f6dde44da70bf/regex-2026.9.29-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f7f7aa47b229f2b39a2ae2596d2ad5625d77b5eb9856fac2dab3eb506cdd0a0", size = 807176, upload-time = "2026-09-29T00:47:47.372Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/4383eed7aa5aef67616cb1b3f3ad06b7c624c4e6cced48630cd5ce133d85/regex-2026.9.29-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d9b77b25b4f395f92de6099ab08e8ae2bc7e51dfe157f22900902243a5cc90c7", size = 784315, upload-time = "2026-09-29T00:47:49.518Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a6/0086ad31cebb183c637d3198547075aa493afde308e1ff61fccccb29ba6e/regex-2026.9.29-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34b6925af9853bf461950e6508910f179fd6e9b1a7ec8548e069606b7e51a26b", size = 793748, upload-time = "2026-09-29T00:47:51.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/a0/f9005cba3f629a859573fc5d1224ea4e1f97919ec8581d018e03a351a604/regex-2026.9.29-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:addd736a0547d553283adaf4e05d7104e7f2c7b0b092e9b4d28756825f14531f", size = 870302, upload-time = "2026-09-29T00:47:53.368Z" }, + { url = "https://files.pythonhosted.org/packages/01/4f/e1a3e46bb5315a4e18b01a990e7a28e2a16595609d50c442baf2815a3c65/regex-2026.9.29-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fe3fa1dd453ed5c7f5ea23a26218329790ed7197a99b90e94330e313959a7f52", size = 770299, upload-time = "2026-09-29T00:47:55.606Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fe/f303b4acfda44e1ff1379368748c1ef2dad04a6a8e9c0ecbc970b19d97ca/regex-2026.9.29-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0cc63b5e47c12a48d90c7e9d7de6a035dd14f62868aaedbb4e0ff8ba2b8bfe7b", size = 861570, upload-time = "2026-09-29T00:47:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/60/b6/b4f7e99249f596017c60ccad5faf9310fc8e3e59bb2244940a90a1b0bdff/regex-2026.9.29-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:724184b4aafed865e4f13ca313fdcb43024300c028ec67319cfa16847d84685e", size = 795967, upload-time = "2026-09-29T00:47:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/fc865a4638d9f6762192b6bab5b7aa1f33a90e9e99578c2e111e2a63c8c3/regex-2026.9.29-cp314-cp314-win32.whl", hash = "sha256:c6c8fabf1dafc1f1ddcbb67896d3f93efb092e8c4b6322d7389b944e76a484e5", size = 274758, upload-time = "2026-09-29T00:48:01.8Z" }, + { url = "https://files.pythonhosted.org/packages/31/e2/c2b466924ccbeb874862968ca638051b15a8fd29d994a0e99004a5cbf78e/regex-2026.9.29-cp314-cp314-win_amd64.whl", hash = "sha256:1c2a0026062abcc321a53db4a185ceba0b59a66b5d37b0808917a88b55a5257f", size = 283817, upload-time = "2026-09-29T00:48:03.614Z" }, + { url = "https://files.pythonhosted.org/packages/c6/42/ea0f8dbaa924fa75c6338935eaee2f44dab369b27f02db1e03d74344b049/regex-2026.9.29-cp314-cp314-win_arm64.whl", hash = "sha256:121a76a0985db80ceae9e171c337f8c927868e37d01b54e3ce87bc87f9c6a208", size = 283663, upload-time = "2026-09-29T00:48:05.624Z" }, + { url = "https://files.pythonhosted.org/packages/44/48/d58e5081119f5c223bbb37d2340acde3d069e1df8e8cd166c37502eee4da/regex-2026.9.29-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e31f72490b7c12f7790e1e25c3afffd20503ee1bfb43461d7838b871ff244b19", size = 501393, upload-time = "2026-09-29T00:48:07.833Z" }, + { url = "https://files.pythonhosted.org/packages/72/3c/c49945287d4f9efee7d41f98072f8ad880efb8f430595a612fbdea996a4e/regex-2026.9.29-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:80ea96f5c1a30bf09007d48466521d9c294bebe197c708c3359096e3e3691632", size = 298237, upload-time = "2026-09-29T00:48:09.684Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1f/688cb61c3d4cf7bcc1ed444b5cc49399eba3e51c469ae285cf87fea3022e/regex-2026.9.29-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:554bffadcbcb6d5f4e5fb10a61cc52084b9a63d1dab5f10bcd2c4343972e8e2c", size = 295936, upload-time = "2026-09-29T00:48:11.454Z" }, + { url = "https://files.pythonhosted.org/packages/26/a3/de43ac6b877b7d09c19a3a426b1bd5acdd209eaaf68f406466f80439ccf6/regex-2026.9.29-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:864e9b87ac33c3fb9fb4ad48166d4fdb579c351d5c77deb0d34bccb36a775cd9", size = 816905, upload-time = "2026-09-29T00:48:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/62/14/9940763201c51d537786304984c67d0fc3d2ed18837ffb6f09a869f6b6c9/regex-2026.9.29-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:044265d77d94f5e3cb2fd72c76723807c429cb8c533e9d4672d0334a6f14f588", size = 881527, upload-time = "2026-09-29T00:48:15.313Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e1/c842d8df0b23245ebf202f8ab9c39fd48e2db39959454ec39a41c8c72082/regex-2026.9.29-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2089fe39c406784d90101c726755ffa1497bb74638fd434300d2b88006186de8", size = 923115, upload-time = "2026-09-29T00:48:17.328Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c1/98622479e3c354a446a75232e522d747d2b3df23092dcd8a5309380a2020/regex-2026.9.29-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0def9fb6abac55492d6d51cddb7225d07d6f279e774e0adc08569a54a5fc8d46", size = 820674, upload-time = "2026-09-29T00:48:19.32Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d0/5808c95f9c79ed27b5eedaafc3df6239ec56a49f2e23ea8f831b18427c82/regex-2026.9.29-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:888d60953908dcf761aa320c3e390ab8556efbdb551ace63921de90f6ae0848d", size = 793306, upload-time = "2026-09-29T00:48:21.615Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d3/021ca2638671ad20603bcd9b4d5bfa35d2610cd216a043ea7f0b44ea39f6/regex-2026.9.29-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed511a0708e2297e1d6431e7fb217e3402791e491e02da800658ace4973df1bb", size = 803103, upload-time = "2026-09-29T00:48:23.871Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2d/755c6d13ef9c657378013676c391c7a402166b3f419a464a3e058dcbe533/regex-2026.9.29-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e1172147d28d8fbcf8cb8d26c41506169f5ad8fe9ec969cb116835a19d4d8eca", size = 872175, upload-time = "2026-09-29T00:48:26.255Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fc/e1cab183b9dafe8597f58c1c766da9bf96204d3b2f232bcf3eeb75ff7b6c/regex-2026.9.29-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:92f05c9c42bde5785dc48770bc2194d9f7442544156f951e19cd31b096cec562", size = 777362, upload-time = "2026-09-29T00:48:28.389Z" }, + { url = "https://files.pythonhosted.org/packages/06/7c/e10ea17fba31fb4a1f9d13ed53a2d2a9066a2aea58d7557e263f6d99e7b0/regex-2026.9.29-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f37964e4a5e993d2fd45147741e9dff7f34a2d8c00ab94c4ea0514a4677f959e", size = 865606, upload-time = "2026-09-29T00:48:30.4Z" }, + { url = "https://files.pythonhosted.org/packages/8e/6e/69824d9aee1fd41c54ea7264654a47c8d9d84d8a228e11c2bcf4c201ed81/regex-2026.9.29-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:951733b1bbdb71e377cec567b409f1a7881b47cfcad84121aa74cb575fa425ea", size = 807945, upload-time = "2026-09-29T00:48:32.375Z" }, + { url = "https://files.pythonhosted.org/packages/89/22/857050a86e21ce60193e02a8ef662521f2e263a645c8b1b905fc136b61a7/regex-2026.9.29-cp314-cp314t-win32.whl", hash = "sha256:65b408d8fcb273e3499e7ef2ce796810da1becd208c7fb4373692a242d79d461", size = 276758, upload-time = "2026-09-29T00:48:34.72Z" }, + { url = "https://files.pythonhosted.org/packages/4d/96/56808fe029553d7d4c703414f2a527faad2ea2bfa9ca094a2e7f8762b530/regex-2026.9.29-cp314-cp314t-win_amd64.whl", hash = "sha256:bf48516e35cf848390ea68850aba53e7c333720d2945b4d2c25b69fc5171723f", size = 286527, upload-time = "2026-09-29T00:48:36.864Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/074e2cfb3d8101a6a764aba5f7c5d1e21de087483e35bdc0c4ce2eb60364/regex-2026.9.29-cp314-cp314t-win_arm64.whl", hash = "sha256:9173db3be74a35cb6731701094b98120f7ee4876a287882a59cdea1fa7da342f", size = 285954, upload-time = "2026-09-29T00:48:38.901Z" }, + { url = "https://files.pythonhosted.org/packages/a7/dc/d84990386c9dfdf8c377f00f371b241fdc9a2c8aea0e3d66941b2e51be0b/regex-2026.9.29-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:c3589f40749acce747510bf5d589d54e376cb0930ea58b35effac97e5312b0c1", size = 497836, upload-time = "2026-09-29T00:48:40.858Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ab/a569ebde875fa12ff8c6c9a30e07503620f195e4be4d54c3d3ee8eecc283/regex-2026.9.29-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:32ab11df9677ca80bcbb5fe4eb1da9109a5019239a054836efc6fa1c64e683cf", size = 296244, upload-time = "2026-09-29T00:48:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3e/7d548e82a108e7c8b2d5246650e397a2f8db599f9b2e975466939c5b4e70/regex-2026.9.29-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7c03031610e3e6ed1768a2b7a8fc84637c1257b50c5eacaf094c6e17a84fc563", size = 293748, upload-time = "2026-09-29T00:48:44.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/34/a8e19a52f452bbb07b32a2bef70dcdf90c2737049749f74cc12d7486fb4f/regex-2026.9.29-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:42e82e578c904445d4c8a35b8f28052cf567593215fa5db06266fbc6f77aaa2e", size = 807840, upload-time = "2026-09-29T00:48:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/7b/11fbd4640b3bb82b72822a63c20ade4013d562d291703a9debeedc24e682/regex-2026.9.29-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0b65c72739f981377c9c22e0c5c3cd7f42da7bd8a3c9209330fac772c7d893ed", size = 879330, upload-time = "2026-09-29T00:48:49.168Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de58c74f1f4e31586d83eb39c56872d686c4e0d0966d151884c833b94ced/regex-2026.9.29-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4408b2b27a95ca8cc48b7411945753773353b5c93b307754781086c99d3a576f", size = 919251, upload-time = "2026-09-29T00:48:51.322Z" }, + { url = "https://files.pythonhosted.org/packages/81/42/a8c480f6dd5ac59fa28ddae79afd9d7ac7e596fdb61813adc65bb6e674b8/regex-2026.9.29-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a714befaacbd10092ffe4cea0d3c5f008fb9efe9bc322c715bcdfdee414b9a3d", size = 808808, upload-time = "2026-09-29T00:48:53.529Z" }, + { url = "https://files.pythonhosted.org/packages/68/60/0bc0d1ec8b37ad64be6fa30e035251f11de9667a0fac9e82ee74517d81be/regex-2026.9.29-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33026515aebc0e70d1c89978e53e8d695d35d9e472f8d5b34465ba3c74028650", size = 789907, upload-time = "2026-09-29T00:48:56.036Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/116a3ef19b3acfe81077f0bf2cbc7714a5e94bc8935b7243ab61cb0f1c3c/regex-2026.9.29-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:31b003f9a070335e2a8233ee9b14a3ca8e6d792012ae011f741bf0aaf11744c5", size = 795770, upload-time = "2026-09-29T00:48:58.284Z" }, + { url = "https://files.pythonhosted.org/packages/96/ba/e38c3f203e7e7e18c957d48e6cb6dbf96c11e95a44efa4a480522afc5d6d/regex-2026.9.29-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:c03c6eb6ece86dfdcbb34799efaa339b093132e1aceed491ba5e08fe06cdf699", size = 870671, upload-time = "2026-09-29T00:49:00.506Z" }, + { url = "https://files.pythonhosted.org/packages/2f/0f/9ee0b0cb76c55f63684bd7fff554978e8773b4fc86e2bcb2d50772dc1086/regex-2026.9.29-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:a5300757f8a68f5b6cc33f57338d72a0e3589c5cc9ad5f8504ea06f028be582a", size = 778631, upload-time = "2026-09-29T00:49:02.984Z" }, + { url = "https://files.pythonhosted.org/packages/b6/19/e6e3eeb226af5872c4958002f6edef4e4f40ea4cc5f5665023f2019eb045/regex-2026.9.29-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:80c7cadd3fd2bfde5df8aa0787e315812cad0c313a753095d02f4c2b6c01677b", size = 862187, upload-time = "2026-09-29T00:49:05.264Z" }, + { url = "https://files.pythonhosted.org/packages/5b/62/823c102e106bb2711d6b7dfe5981552fe4467b2969c46a20c5c383cf498c/regex-2026.9.29-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:3f1e6cb402a89457582cd696f982559217d13484a193202c394015297968c86d", size = 798423, upload-time = "2026-09-29T00:49:07.644Z" }, + { url = "https://files.pythonhosted.org/packages/37/e0/e927776258fa70b2f6feffc3be584ffc85ba4c1e20a320f0aee9a632fc7d/regex-2026.9.29-cp315-cp315-win32.whl", hash = "sha256:a64b85a4760337cfefdb27d42da6ed8b58e8cde3f2d57b6ef43e76ef6ea9ef47", size = 274757, upload-time = "2026-09-29T00:49:10.513Z" }, + { url = "https://files.pythonhosted.org/packages/77/04/358de85d1860238e1b4fa98fc2c80c990124a25d2e14739e28cc02c25562/regex-2026.9.29-cp315-cp315-win_amd64.whl", hash = "sha256:b3e445b66c80b4eb4234e855ce94d9adc183eedbd632816228d89930b91b2c5b", size = 283824, upload-time = "2026-09-29T00:49:12.849Z" }, + { url = "https://files.pythonhosted.org/packages/92/d3/d5c5b264784a5ab2b0f8cf620c1eeb4dbf3440d306761905e7d99345bef5/regex-2026.9.29-cp315-cp315-win_arm64.whl", hash = "sha256:8f39588af4731c8923c26810eb3b33f76f17633985e40f59c3cd45a33805a895", size = 283664, upload-time = "2026-09-29T00:49:15.331Z" }, + { url = "https://files.pythonhosted.org/packages/02/dc/f63ec2c201445ce1150fe780f5c56f16a10124d9a9da3a93161dbb0d8892/regex-2026.9.29-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:fb99cc9d45f48895d9d67f6a0b8a57f08d39c174d9f25ad97a313e0470267b1c", size = 501549, upload-time = "2026-09-29T00:49:17.705Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/d2a698dc6bfc11fbce03f1cb0249c13284e93b79ed11f893edf6fac431c9/regex-2026.9.29-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:720537c7ea6f80dc61913184edb0ce2497a306b39ef19f28505b322553d52bdb", size = 298187, upload-time = "2026-09-29T00:49:20.171Z" }, + { url = "https://files.pythonhosted.org/packages/85/b7/88dcdb38cd3935d4ee9e9ce9b8e56cb3b3518d1f020acfa7dd62ad289bf8/regex-2026.9.29-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:0fd2c901cc307a745ad4bc87f20060d7a0825a3371d1e93488af22e7a387f78f", size = 296157, upload-time = "2026-09-29T00:49:22.342Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8e/ba6c01dde33a69fc294b38b43f6677baaa5735a6248f39708031a738158a/regex-2026.9.29-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b11b589e00095ec69cf79841a76360f9b079e95b0368a25b5ebb951ab0c157ff", size = 818468, upload-time = "2026-09-29T00:49:24.612Z" }, + { url = "https://files.pythonhosted.org/packages/2a/f1/2586693e3a2d6b1247852593d37a6c17b42a92ee44f7cdcb9a0c1494e64a/regex-2026.9.29-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d7cab119d0df0b9413f106b4d7fc34f2872d3574ed3806fb48959c830b1537da", size = 882825, upload-time = "2026-09-29T00:49:26.996Z" }, + { url = "https://files.pythonhosted.org/packages/30/51/084f3e7bdcd0e9c33665c938cf5d134dc3548cbb4a75f0197ec7bfd754b1/regex-2026.9.29-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b89efc38431793d28b7cd91227e2f952ad7c48df19132b17f43a5fec3c14143b", size = 923314, upload-time = "2026-09-29T00:49:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f1/066c6fc23b7dc229789c21c880b5ba5ad689fb95fed12e078266f55a1f9b/regex-2026.9.29-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80a5ea3b4fd9d6a5b9a44f7976a9acaaab35aa3c1f6b29e5bd857dfabaded223", size = 822222, upload-time = "2026-09-29T00:49:32.404Z" }, + { url = "https://files.pythonhosted.org/packages/0a/56/592cd46fdb8f2f8682a1d7fd1310e4d0bcb93fbd0e6bbe4141ac28240227/regex-2026.9.29-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:19959129885356df0e97556856f77eb2888380dac18bed075a7c05c5128c618d", size = 794882, upload-time = "2026-09-29T00:49:35.076Z" }, + { url = "https://files.pythonhosted.org/packages/ee/4d/d65384bb071c864b01aa8314e3a6a687845ebd57588390976edc960c218b/regex-2026.9.29-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6a1a824fbed817e0a891103886b68f063b1e83cc51bc97192a90a60195a9291f", size = 805887, upload-time = "2026-09-29T00:49:37.395Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/358de0d8f40d5178e4f7e7e121cfd5b961c812b77a055d11f5079e3f8fd7/regex-2026.9.29-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1ba8c6a416569ce0d37e83e28a254a61dc99a419084dfb6476cea02d997f74fa", size = 872901, upload-time = "2026-09-29T00:49:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/00/06/6bfded72d043240c6b52bbb5e16f639d81affbf7484b4fe2ec45f3d4afc9/regex-2026.9.29-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:446654b29bfaa30500d80947eda42cef1449dc8a87f4e3cf061cc8485d3a1f0b", size = 782970, upload-time = "2026-09-29T00:49:42.581Z" }, + { url = "https://files.pythonhosted.org/packages/5a/20/9f418a50baa78b3ed8308fcb0cc49e472dd000b7ef935a7295af202ea744/regex-2026.9.29-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:bf3c49863c23a1ad6da9c30351aed6cff8d5ddbeb63c5c8420ae54e98c7d0138", size = 865441, upload-time = "2026-09-29T00:49:45.238Z" }, + { url = "https://files.pythonhosted.org/packages/2c/29/817c7eacdeaf8463123e949bd394c39ad024eea1ec38ddf5ad141da2f3bd/regex-2026.9.29-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:01000ddf0e3ffef97f2413ceb514f6313040106b6d18a03ee00a4fe35c1eb1db", size = 809431, upload-time = "2026-09-29T00:49:47.878Z" }, + { url = "https://files.pythonhosted.org/packages/63/0b/83aab3b5b739947f744135a7a3a446e25433ebc92b05e01aae197ccbfdda/regex-2026.9.29-cp315-cp315t-win32.whl", hash = "sha256:c4e38dd8f39c43a91d2410ad2b85610701b0979342c3df1d69eaf8e838c757d8", size = 276964, upload-time = "2026-09-29T00:49:50.524Z" }, + { url = "https://files.pythonhosted.org/packages/72/f2/6314b5fc68789b5dcc38885bc6e3d6986b34fb3372b7231088ee5cecaa05/regex-2026.9.29-cp315-cp315t-win_amd64.whl", hash = "sha256:e2c89e9b762c57f59d5e99ee8b20202adb892e35f8d3485741340999ca55058e", size = 286487, upload-time = "2026-09-29T00:49:53.224Z" }, + { url = "https://files.pythonhosted.org/packages/56/bc/97b2245c8c7b2dd01f2db74f2bea003cd33c15009b4996a2447f46b5325c/regex-2026.9.29-cp315-cp315t-win_arm64.whl", hash = "sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34", size = 285955, upload-time = "2026-09-29T00:49:55.655Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "ruff" version = "0.16.9" @@ -1215,6 +1670,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + [[package]] name = "scikit-learn" version = "1.9.1" @@ -1337,6 +1816,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/ad/741c19fcb66755ff953daf9243af8480e4bf3d7fbe57583c178c7d2b6b51/scipy-1.18.1-cp315-cp315t-win_arm64.whl", hash = "sha256:eda632a7981f69730d6281f451db9c1c370993a2c0d7ddb43e2a809a2862b83a", size = 25319710, upload-time = "2026-08-21T23:28:45.713Z" }, ] +[[package]] +name = "sentence-transformers" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "tokenizers" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/a1/53ae87971817e2d8370f8e79b843a881be33ab339502d71c5f82ac31f7af/sentence_transformers-6.1.0.tar.gz", hash = "sha256:299025df51550dc1a38f05be27a9b0bf881c4e5e70542b3b7757d05e00aa3868", size = 576566, upload-time = "2026-09-18T10:44:24.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/6e/9115e19589c83172bd37ec793cfef7f7c719464053fa6a8882700c1f4488/sentence_transformers-6.1.0-py3-none-any.whl", hash = "sha256:eb8122f4d180f552eda26dc3d77e84e8c11dc2b1d456a406b9f24abb70ceeadd", size = 740560, upload-time = "2026-09-18T10:44:22.841Z" }, +] + +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1440,6 +1957,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + [[package]] name = "threadpoolctl" version = "3.7.0" @@ -1449,6 +1978,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/3f/f88a53f60a472b46f4023f56d204dd7de33d34c5d2acbfa0d70a674e639e/threadpoolctl-3.7.0-py3-none-any.whl", hash = "sha256:cd8b60b5641b45c67bbf73c64c843235fc2d8a480c87389f52f5dbee893b86be", size = 26362, upload-time = "2026-09-15T15:46:19.168Z" }, ] +[[package]] +name = "tokenizers" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/1e/bc6587c5ab643b2e17776cace9070a2ae73549c86bffac9934a600bf3c31/tokenizers-0.23.2.tar.gz", hash = "sha256:7f0f085686b9de0d0079e6f874ae053600db64c5d13049e0bbc0119926d25aac", size = 385745, upload-time = "2026-09-03T08:55:42.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/ed/8a443528baa6fac8dfe8c3b75b038c63ac92bb539bcabe311e227c718173/tokenizers-0.23.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:85a9a357a3764aecc904ee76bdaf8cf1ad8e5a67a1b929a487c4a39b49ed0e90", size = 3148852, upload-time = "2026-09-03T08:55:30.874Z" }, + { url = "https://files.pythonhosted.org/packages/67/49/22da045a91732384d3a3771816bf188dc5a1f702c32e635afa7c679c0bef/tokenizers-0.23.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:986670e43691469dcee610ea0f846f91a8f84e91fc6f7a48d4c064414c0ec2bf", size = 3101593, upload-time = "2026-09-03T08:55:28.587Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4d/8f569ed49372a3ed8e57099bd515055fd48d7c95912c4307cda6973c2168/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a37039b5dfc4af84eb3ef0a92f4307e28936c8f9adccba2629d36f652e9bf7a2", size = 3516830, upload-time = "2026-09-03T08:55:14.741Z" }, + { url = "https://files.pythonhosted.org/packages/2a/de/e2f14c8919d5bf51874051d00d6c7b7e0e8bde6c6a2dbeddda7f642896ff/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b7e37ba198f24150f523e1242e83c4970de4a525480586be5dcc24d9add32c5", size = 3407975, upload-time = "2026-09-03T08:55:16.842Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bd/93c69152d02ef06ce47aed8b2bf4952dcf733c935a62791873932b2934d9/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43e4f2071e3cc8d5d86421c874aebc82659bb51a68bcdef5a0da75ee89511ccb", size = 3748165, upload-time = "2026-09-03T08:55:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/2d/b7/56b84b80bc96942bba8eb23751a9e8a1fce4faaf4390425e7083f721c98c/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:325fee2e0418a9dc6c9ecf736a5f5f0db7875183ace9549ae339da76f7a1fbb7", size = 4024165, upload-time = "2026-09-03T08:55:18.806Z" }, + { url = "https://files.pythonhosted.org/packages/9b/8a/0175e216f005c2fe08238292663aa41e4c802b216e71047a69a0e9fc6fa3/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:950d7c9426fa72406a0ffeacdbc0bb9985f5db20eb8b263f29c79aaf83105703", size = 3591899, upload-time = "2026-09-03T08:55:22.752Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ca/ca6b93c7820df123b2662a9469e8facc826ccc94e98fdd0d615f6431e73a/tokenizers-0.23.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41c2f84d172449b4dadb9cdc508e3e364076613c35b16e76ecfe47a60d1e3305", size = 3386843, upload-time = "2026-09-03T08:55:26.584Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a4/4f9106d317b14a80aefea9f0e3a8d07ef25f856a7607eb7f5ab894281fcb/tokenizers-0.23.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:12f0835dc2ee694746a76adf7b1567d4346a4a502ebe93fb1f5f80ea49799b78", size = 3577314, upload-time = "2026-09-03T08:55:20.825Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6a/1552b70fb0d9ab074fd3fc961435d01364e79c9058481822c3af6e8d402c/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:eb2f9c8a24da020ea8c11a01a19c1c2547912d92121ae4a01cfbca46125dee40", size = 9967367, upload-time = "2026-09-03T08:55:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/06/01/3ccb3a956c7528b2507b8a9714155c4baf86af593039db6ea375dd0c96c3/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f486f402f6f9abee5bb032553736813af0c710a86b2e0ca592634c55cea1f835", size = 9811886, upload-time = "2026-09-03T08:55:35.642Z" }, + { url = "https://files.pythonhosted.org/packages/fa/73/7038e612d48bda1599457f712f6bd3854eae1a9dc9c13aa47f835349db48/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:bef235815a067b2648caf6dcc7a71091b0b0fff9ee8057f6451eb9335fae52ef", size = 10146224, upload-time = "2026-09-03T08:55:38.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d8/8e9e4e0b287a338d8f88976729628c9d22e8a54cfaf9777018a7f7cb58a0/tokenizers-0.23.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c56bda1511921587789163e524d196ed8284174ac23abd7685d5ea8da6c4718", size = 10256304, upload-time = "2026-09-03T08:55:40.977Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1f/c79a01f671a49728ebb0b61f7ff9ea45663b66cab40bc0858e9859b25c16/tokenizers-0.23.2-cp310-abi3-win32.whl", hash = "sha256:debf978920d93ba9c219bd67cc4bbfaf912c9039e41e7a28b91ec15e3728c95a", size = 2592809, upload-time = "2026-09-03T08:55:48.02Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/0a69ac6b82dbccf3f71add938a161c497952749294b8dd6dfe03a819dc40/tokenizers-0.23.2-cp310-abi3-win_amd64.whl", hash = "sha256:2e96f5699d5249c9c64aa8412e044f727aae3a4098cf830f9901ec1afc361cde", size = 2863236, upload-time = "2026-09-03T08:55:46.193Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b0/dee84cb44175be1b4c35bd2f770727494e78f0bb38e571a623ade94dbebb/tokenizers-0.23.2-cp310-abi3-win_arm64.whl", hash = "sha256:e49c394456dd9985787fec76132438ba3fb8911f857b1bf3d40119f9292d41aa", size = 2729352, upload-time = "2026-09-03T08:55:44.345Z" }, +] + +[[package]] +name = "torch" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/3c/379349820e9a6e713826de3d0b88f07728541bdcabe2a3afd128871cdc4b/torch-2.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:420dbf314c180ee4b86e9bc00aee5746a7d6e5bacdd7df925af671ed393f0b2e", size = 127307099, upload-time = "2026-09-30T17:44:10.421Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3e/474930ab971e306957ec763b7df695d659179e15ccab9e7073d47ae14ce8/torch-2.14.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0b13acde294b401509830d85a8906cbf2fc97425a12a66cce1d9fa13c5106a2a", size = 453998918, upload-time = "2026-09-30T17:52:45.134Z" }, + { url = "https://files.pythonhosted.org/packages/ba/fd/c470857d0dfc5d4c94430658b3eed60fd5292b0bf596a8aae7b9fd66d674/torch-2.14.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:23011fe29a99b591eabb3a31a25080c62e2c2f0690a3c798744e5489dae50451", size = 554618695, upload-time = "2026-09-30T17:53:06.8Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1a/b9648b0d228e513f69f8453ec3f90a8d436d78ff6d85a2773e539f0c07cf/torch-2.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:38bee9f2a2ccfc6898172a5075e4ba52522fbe143eeb099674fc67ab390e858d", size = 124113906, upload-time = "2026-09-30T17:52:21.377Z" }, + { url = "https://files.pythonhosted.org/packages/7d/11/faaca4f8541c45127b7e0d6bb141221fe944c8466d89986b8466c6c195f9/torch-2.14.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:dbe359d705f4d67236743794c296c6dee93a922fd8117eff8c3e880d7d0fb2b9", size = 127315286, upload-time = "2026-09-30T17:52:27.43Z" }, + { url = "https://files.pythonhosted.org/packages/60/1f/0330275c705b846882531c64e80d32b13fb572eeb9b995e87e84154b0712/torch-2.14.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:6d530bd11479fb574567af3a1f384af9a77bf5bb9550cde8080b356ca7220d5a", size = 453999200, upload-time = "2026-09-30T17:53:21.716Z" }, + { url = "https://files.pythonhosted.org/packages/7c/cc/bb579ac0c80e077c58204e43b37a254b74dea5873635100bea25789a55d4/torch-2.14.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c8f71aabc67bcbfc9373dc131537a5968d04edce73e88add21354a7cd0a76985", size = 554618164, upload-time = "2026-09-30T17:53:42.73Z" }, + { url = "https://files.pythonhosted.org/packages/7b/4b/32c00cacbe682a4c08d9c6c3e7a7116c912184cc7155f06c64f545db97d1/torch-2.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:711713391d26a1ce5e9fbc6c996d954a8c12e8825374cc77b809a6af29539b8c", size = 124113754, upload-time = "2026-09-30T17:52:33.474Z" }, + { url = "https://files.pythonhosted.org/packages/43/19/23a1aed488423a5055727256b25406e4b93bd2bcf1352bef582b9951c10c/torch-2.14.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:cee091caf2a6229e248daf41d18ceebf590ba02d9179205610062c64ee5fef03", size = 127325330, upload-time = "2026-09-30T17:52:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/93/f4/94219ada13edd62fda1f976163292b7b8595f1fd2f4ef74af24b3baad375/torch-2.14.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f68f5476e2bc0e8f60b74f7ca21acda885c4477af6392f8977e4f0d1ea1aa162", size = 454016297, upload-time = "2026-09-30T17:54:03.078Z" }, + { url = "https://files.pythonhosted.org/packages/fd/df/23c69e9b9fd19fe6563422f1bc59f89601bda47234a89a1cf432cfde3aee/torch-2.14.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:1d4df07be7338bbcc4d54085adee21363c91314702b9bd4d78ef72ffac9465ba", size = 554621744, upload-time = "2026-09-30T17:54:22.951Z" }, + { url = "https://files.pythonhosted.org/packages/d9/dc/a36a4431ab5e3ad168a75f341e24677a4bbbf0c5dda59c97a7079262bb94/torch-2.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:d02a4c48a2ca5fb7654e36e71f710f74494d83f1f10aaff8e059dd554adca956", size = 124110781, upload-time = "2026-09-30T17:53:31.141Z" }, + { url = "https://files.pythonhosted.org/packages/2d/bc/1afbd1a22f6023eafcdb9295ac1d97137a7816f54091d693701072da2218/torch-2.14.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:06c3ec25f3b497f9a73dc0c36f293f295cfe8d5584fd446238d9a501b30a66d5", size = 127657364, upload-time = "2026-09-30T17:53:53.837Z" }, + { url = "https://files.pythonhosted.org/packages/e4/94/b97e863c9ceef2bc6e082e967cbfb9065b5e232f2a60c5af0f5115a1bc31/torch-2.14.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:9113f94f70429f9f302bf55b090b5411269083e8a72a5d015ffcc5a7f83f2c69", size = 453998592, upload-time = "2026-09-30T17:54:38.268Z" }, + { url = "https://files.pythonhosted.org/packages/03/d8/8272157c438cc26a199a8fa0caf87985477b8a1ed8a09084eb2ae40609d8/torch-2.14.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:e65d5fe136e533b23c2d134377f7f126721c7c78dd75b3dbb735af082a8aeb85", size = 554578730, upload-time = "2026-09-30T17:54:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/45/05/451a69a4287033d8f106f5c65f92c2c0c37229d81ea101d4b047caf758cc/torch-2.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e07306caa1de2a4ac1467e11ecfc92fc44f523dd6a521145039aef46d913963c", size = 124400129, upload-time = "2026-09-30T17:54:12.306Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + +[[package]] +name = "transformers" +version = "5.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/4c/70012ed0795235fb6bd7806fafd4a1842f03e49ebbfa3bb9580e451948ce/transformers-5.19.0.tar.gz", hash = "sha256:87f38dd25e4521151b97e94520ac457f44a0ae8a8358a5b112daff6c64a822d6", size = 10032022, upload-time = "2026-10-06T16:39:00.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/59/1e7f1212c215c73690ae6cab4678efe1555ff985953e145d88cc212859b0/transformers-5.19.0-py3-none-any.whl", hash = "sha256:afcd2dd5f603ed28c1e1fcb00a338ccbb4ef5f878ed289635df8b58187afb518", size = 12599123, upload-time = "2026-10-06T16:38:56.794Z" }, +] + +[[package]] +name = "triton" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/07/0f8cd8e8db0472334253efdaaab3d0819fea27aa99bf0e7f1aeea4ceb5ae/triton-3.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9c404c69ed4a39e8ec632eaf6b9fe058a060bf98979c177f6ef666f06bb8d50", size = 226474486, upload-time = "2026-08-28T16:08:18.29Z" }, + { url = "https://files.pythonhosted.org/packages/c1/09/b7012e5bfae67640f268aa584caa80fe1674f6b0da949046b679972c33e3/triton-3.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91ffa46d095b252248297292dd22bcbacd53a125a0c2eefbbbf74925a320bc3", size = 247972921, upload-time = "2026-08-28T15:55:53.157Z" }, + { url = "https://files.pythonhosted.org/packages/87/4d/4c564374bcdadb166fccbf3e45aee0d4a473f88d341761bd2fefe3b8e8c1/triton-3.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7004666652f500ed854a86988e4b3d69d247188b5d2092b5df1e44f4a954099", size = 226476793, upload-time = "2026-08-28T16:08:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/3394d5548404c1cabd1dadadd28d0b3f9478db1dff8180da53bb3f0a1e19/triton-3.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f0497218e26b7d79773ad9c2a3fa3b539ee69f587a13fac2e552b1d322a8015", size = 247975122, upload-time = "2026-08-28T15:56:04.112Z" }, + { url = "https://files.pythonhosted.org/packages/b8/59/bf0e9493118bb353ab59a5d6a65db3618d9b314417cc1459f0121e0ec5c9/triton-3.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f6b48d0591929a3867973acac3dccd4e058585f91bfb41022de496c9ffab304", size = 226488654, upload-time = "2026-08-28T16:08:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/93/d9/08c75f3459f19ad00425b564058e40efa4bcd79b816064cf27499303ea42/triton-3.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:387dae4cb0089a7b6ba1a428ae0782b65c4c58f57d94617cb22ca8593d8ccbca", size = 247972313, upload-time = "2026-08-28T15:56:14.007Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/429c5592181cfb7361a0a8e0bff218e7224b726709d75da2472b3e819f70/triton-3.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b84e7d512490ba529111260fa6f7cad8b254a6bb5fbdf41d5ef9a5e57f52d0a", size = 226591133, upload-time = "2026-08-28T16:09:02.271Z" }, + { url = "https://files.pythonhosted.org/packages/fe/d1/aa8a3e935c37efee7945984fdb64d7e0851bf6d920afd97b2d21f9d23360/triton-3.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74217bb56ed8692759227758e4c4b3bd2d608a209c1a7a081bf361fb4c2c1bf9", size = 248077577, upload-time = "2026-08-28T15:56:24.94Z" }, +] + [[package]] name = "truststore" version = "0.10.4" @@ -1458,6 +2100,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] +[[package]] +name = "typer" +version = "0.27.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/51/d33db42cc72ffd8c30777547b42d01f0cbf9d95a770457698d0174b3ed71/typer-0.27.3.tar.gz", hash = "sha256:d0396f770a560ab1b0a8504e13b5f254b728cedb05c61cf0359e944e50ce8901", size = 205303, upload-time = "2026-10-06T17:24:16.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/ea/2e31b67051e91a133189e9c000c222502ddc6969856416de0d095de4c0b0/typer-0.27.3-py3-none-any.whl", hash = "sha256:e50022f28b82a86313e54501317a1db64bf8f8d036ff8cfe5ca7e47675454aff", size = 123312, upload-time = "2026-10-06T17:24:15.054Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..d71d200 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,33 @@ +# TLS in front of the API and the console: `make tls` (docker compose --profile tls). +# FRAUDLENS_SITE is the public host name. Left at localhost, Caddy issues a +# certificate from its own local CA (expect a browser warning, or trust it); with +# a real domain it gets one from Let's Encrypt. +# +# Caddy is the edge: it drops any X-Forwarded-For a client sends and writes the +# real peer address, so the API can believe this one proxy (FRAUDLENS_TRUSTED_PROXIES). + +{$FRAUDLENS_SITE:localhost} { + encode zstd gzip + + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Content-Type-Options nosniff + Referrer-Policy no-referrer + X-Frame-Options DENY + -Server + } + + # Nothing the API takes is this big; refuse it here, not in Python. + request_body { + max_size 2MB + } + + @api path /v1/* /health /ready /docs /openapi.json + handle @api { + reverse_proxy api:8010 + } + + handle { + reverse_proxy console:3100 + } +} diff --git a/deploy/k8s/base.yaml b/deploy/k8s/base.yaml new file mode 100644 index 0000000..ddbc215 --- /dev/null +++ b/deploy/k8s/base.yaml @@ -0,0 +1,162 @@ +# FraudLens on Kubernetes: the API and the stream workers (docs/SCALING.md). +# +# Postgres and Redis are not here: point FRAUDLENS_DATABASE_URL and +# FRAUDLENS_REDIS_URL (Secret `fraudlens-backing`) at managed ones. The dataset +# and the model registry are read from the `fraudlens-artifacts` volume claim, +# written once by `make pipeline` (or a Job running it). +# +# kubectl apply -k deploy/k8s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fraudlens + labels: {app.kubernetes.io/part-of: fraudlens} +data: + FRAUDLENS_EVENTS_STREAM: fraudlens:events + # The API scores requests and follows the log; consuming the stream is the + # workers' job, so the API keeps its worker off. + FRAUDLENS_RUN_WORKER: "false" + FRAUDLENS_FOLLOW_INTERVAL_S: "0.5" + # An entry a worker read but has not acknowledged after this long is taken + # over by another worker (a crashed or evicted pod). + FRAUDLENS_CLAIM_IDLE_MS: "30000" + FRAUDLENS_WORKER_PORT: "8081" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fraudlens-api + labels: {app.kubernetes.io/name: fraudlens-api, app.kubernetes.io/part-of: fraudlens} +spec: + replicas: 2 + selector: + matchLabels: {app.kubernetes.io/name: fraudlens-api} + strategy: + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} + template: + metadata: + labels: {app.kubernetes.io/name: fraudlens-api, app.kubernetes.io/part-of: fraudlens} + spec: + securityContext: {runAsNonRoot: true, runAsUser: 10001, fsGroup: 10001} + containers: + - name: api + image: fraudlens-backend:latest + command: [uvicorn, fraudlens.api:create_app, --factory, --host, 0.0.0.0, --port, "8010"] + ports: [{name: http, containerPort: 8010}] + envFrom: + - configMapRef: {name: fraudlens} + - secretRef: {name: fraudlens-backing} + resources: + requests: {cpu: 500m, memory: 1Gi} + limits: {cpu: "2", memory: 2Gi} + # Rebuilding the feature state from the database takes a while at start. + startupProbe: + httpGet: {path: /health, port: http} + periodSeconds: 5 + failureThreshold: 120 + livenessProbe: + httpGet: {path: /health, port: http} + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: {path: /ready, port: http} + periodSeconds: 5 + timeoutSeconds: 3 + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true} + volumeMounts: + - {name: artifacts, mountPath: /app/artifacts, subPath: artifacts, readOnly: true} + - {name: artifacts, mountPath: /app/data, subPath: data, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: artifacts + persistentVolumeClaim: {claimName: fraudlens-artifacts, readOnly: true} + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: fraudlens-api + labels: {app.kubernetes.io/name: fraudlens-api, app.kubernetes.io/part-of: fraudlens} +spec: + selector: {app.kubernetes.io/name: fraudlens-api} + ports: [{name: http, port: 80, targetPort: http}] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fraudlens-worker + labels: {app.kubernetes.io/name: fraudlens-worker, app.kubernetes.io/part-of: fraudlens} +spec: + replicas: 2 # the ScaledObject (keda.yaml) or the HPA (hpa-external.yaml) owns this + selector: + matchLabels: {app.kubernetes.io/name: fraudlens-worker} + template: + metadata: + labels: {app.kubernetes.io/name: fraudlens-worker, app.kubernetes.io/part-of: fraudlens} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8081" + prometheus.io/path: /metrics + spec: + securityContext: {runAsNonRoot: true, runAsUser: 10001, fsGroup: 10001} + # SIGTERM finishes the batch in hand; what is left is reclaimed by the others. + terminationGracePeriodSeconds: 45 + containers: + - name: worker + image: fraudlens-backend:latest + command: [python, -m, fraudlens.platform.worker] + env: + # The pod name is the consumer name, so a restarted pod is a new consumer + # and the old one's entries are claimed like any dead worker's. + - name: FRAUDLENS_WORKER_NAME + valueFrom: {fieldRef: {fieldPath: metadata.name}} + envFrom: + - configMapRef: {name: fraudlens} + - secretRef: {name: fraudlens-backing} + ports: [{name: health, containerPort: 8081}] + resources: + requests: {cpu: "1", memory: 1Gi} + limits: {cpu: "2", memory: 2Gi} + startupProbe: + httpGet: {path: /ready, port: health} + periodSeconds: 5 + failureThreshold: 120 + livenessProbe: + httpGet: {path: /health, port: health} + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: {path: /ready, port: health} + periodSeconds: 5 + timeoutSeconds: 3 + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true} + volumeMounts: + - {name: artifacts, mountPath: /app/artifacts, subPath: artifacts, readOnly: true} + - {name: artifacts, mountPath: /app/data, subPath: data, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: artifacts + persistentVolumeClaim: {claimName: fraudlens-artifacts, readOnly: true} + - name: tmp + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: fraudlens-api +spec: + minAvailable: 1 + selector: + matchLabels: {app.kubernetes.io/name: fraudlens-api} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: fraudlens-worker +spec: + # A drained node takes at most one worker at a time; the others claim its entries. + maxUnavailable: 1 + selector: + matchLabels: {app.kubernetes.io/name: fraudlens-worker} diff --git a/deploy/k8s/hpa-external.yaml b/deploy/k8s/hpa-external.yaml new file mode 100644 index 0000000..bbbc9f4 --- /dev/null +++ b/deploy/k8s/hpa-external.yaml @@ -0,0 +1,33 @@ +# The same policy without KEDA: a plain HPA on an external metric. +# +# Not in kustomization.yaml by default; use it instead of keda.yaml. It needs an +# external-metrics adapter (prometheus-adapter) serving `fraudlens_stream_lag`, +# for example from redis_exporter started with +# --check-streams=fraudlens:events +# and an adapter rule mapping redis_stream_length{stream="fraudlens:events"} +# to fraudlens_stream_lag. The workers' /metrics also reports their own +# backlog and latency for dashboards. +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: fraudlens-worker +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: fraudlens-worker + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: External + external: + metric: + name: fraudlens_stream_lag + target: + type: AverageValue + averageValue: "2000" + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: [{type: Pods, value: 1, periodSeconds: 60}] diff --git a/deploy/k8s/keda.yaml b/deploy/k8s/keda.yaml new file mode 100644 index 0000000..4ddece2 --- /dev/null +++ b/deploy/k8s/keda.yaml @@ -0,0 +1,45 @@ +# Workers scale on stream lag (needs KEDA, https://keda.sh). +# +# A worker deletes each entry once its decision is committed, so the stream's +# length IS the lag: entries written but not yet decided. KEDA's redis-streams +# trigger reads XLEN (lagCount would need Redis 7 and counts the same thing here). +# +# The measured ceiling is in docs/SCALING.md: phase 1 (feature state) is ordered, +# so more than about four workers buys little. Raise maxReplicaCount only with +# a load test that shows it helps. +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: fraudlens-redis +spec: + secretTargetRef: + - {parameter: password, name: fraudlens-backing, key: REDIS_PASSWORD} +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: fraudlens-worker +spec: + scaleTargetRef: + name: fraudlens-worker + minReplicaCount: 1 + maxReplicaCount: 4 + pollingInterval: 10 + cooldownPeriod: 120 + advanced: + horizontalPodAutoscalerConfig: + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: [{type: Pods, value: 1, periodSeconds: 60}] + triggers: + - type: redis-streams + metadata: + address: redis.fraudlens.svc:6379 + stream: fraudlens:events + consumerGroup: scorer + # One more worker per 2,000 undecided entries (a few seconds of work each). + streamLength: "2000" + authenticationRef: + name: fraudlens-redis diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..291b604 --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: fraudlens +resources: + - base.yaml + - keda.yaml # or hpa-external.yaml, without KEDA +images: + - name: fraudlens-backend + newName: fraudlens-backend # your registry + newTag: latest diff --git a/docker-compose.scale.yml b/docker-compose.scale.yml new file mode 100644 index 0000000..da2a34f --- /dev/null +++ b/docker-compose.scale.yml @@ -0,0 +1,37 @@ +# The demo with the event stream consumed by separate workers (docs/SCALING.md): +# +# docker compose -f docker-compose.yml -f docker-compose.scale.yml \ +# --profile demo up -d --scale worker=4 +# +# The API keeps scoring requests and follows what the workers change; the +# workers share the stream in one consumer group. +services: + api: + environment: + FRAUDLENS_RUN_WORKER: "false" + + worker: + profiles: [demo] + build: ./backend + init: true + command: ["python", "-m", "fraudlens.platform.worker"] + env_file: + - path: backend/.env + required: false + environment: + FRAUDLENS_DATABASE_URL: postgresql+psycopg://fraudlens:${POSTGRES_PASSWORD:-fraudlens_dev}@postgres:5432/fraudlens + FRAUDLENS_REDIS_URL: redis://:${REDIS_PASSWORD:-}@redis:6379/0 + # The API builds the dataset, models and database on its first start. + depends_on: + api: + condition: service_healthy + volumes: + - data:/app/data:ro + - artifacts:/app/artifacts:ro + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request as u; u.urlopen('http://127.0.0.1:8081/ready')"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 5m + stop_grace_period: 45s diff --git a/docker-compose.yml b/docker-compose.yml index 75d49e1..7c5d8f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,9 @@ services: environment: FRAUDLENS_DATABASE_URL: postgresql+psycopg://fraudlens:${POSTGRES_PASSWORD:-fraudlens_dev}@postgres:5432/fraudlens FRAUDLENS_REDIS_URL: redis://:${REDIS_PASSWORD:-}@redis:6379/0 + # Only the TLS proxy's X-Forwarded-For is believed (its fixed address below). + FRAUDLENS_TRUSTED_PROXIES: '["172.30.250.10/32"]' + networks: [default, edge] depends_on: postgres: condition: service_healthy @@ -92,8 +95,32 @@ services: condition: service_healthy ports: - "127.0.0.1:3100:3100" + networks: [default, edge] + + # `make tls`: HTTPS with HSTS in front of the API and the console (deploy/Caddyfile). + proxy: + profiles: [tls] + image: caddy:2-alpine + environment: + FRAUDLENS_SITE: ${FRAUDLENS_SITE:-localhost} + depends_on: [api, console] + ports: + - "127.0.0.1:${TLS_HTTPS_PORT:-8443}:443" + volumes: + - ./deploy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy:/data + networks: + edge: + ipv4_address: 172.30.250.10 + +networks: + edge: + ipam: + config: + - subnet: 172.30.250.0/24 volumes: pgdata: data: artifacts: + caddy: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b8d439b..dc8f064 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,9 @@ evidence, and the deterministic template is used instead. It is off by default. process for speed (about 3 ms for features, models, policy and reasons when measured in process; served figures are in PLATFORM §11). Every event that entered the state is stored with its position, so a restart rebuilds -the same state exactly. The cost is one scorer process (PLATFORM §10). +the same state exactly. The same log lets several processes hold the state: each +catches up from it before it changes anything, so stream workers scale out +behind one ordered step (SCALING.md). **Two ways in.** `POST /v1/score` answers now, for a payment waiting on a decision. `POST /v1/events` queues to a Redis stream for everything that needs no @@ -150,5 +152,9 @@ in order (dataset, features, models, policy, database, replay, review, retrain, shadow scoring), and then serves. Each step is skipped when its output exists, so a restart goes straight to serving. -Not built: TLS, a secrets manager, horizontal scaling of the scorer, periodic +`docker-compose.scale.yml` runs the stream workers as their own containers +(`--scale worker=4`), and `deploy/k8s` has the Kubernetes manifests, with the +workers scaled on stream lag (SCALING.md). + +Not built: TLS, a secrets manager, periodic state snapshots. These are listed with the other limits in PLATFORM §10. diff --git a/docs/BANGLADESH_CONTEXT.md b/docs/BANGLADESH_CONTEXT.md new file mode 100644 index 0000000..c816a86 --- /dev/null +++ b/docs/BANGLADESH_CONTEXT.md @@ -0,0 +1,144 @@ +# Bangladesh context: the problem in numbers + +FraudLens's model results come from synthetic data (DATA_ASSUMPTIONS.md). This +page covers the real problem they stand in for. Every figure is from a public +source, linked below and accessed on **7 October 2026**. "Derived" means we did +the arithmetic ourselves from the sourced figures. Where no public source could +be found, we say so. + +## 1. The problem in numbers + +### Scale of MFS + +| Figure | Value | Source | +| --- | --- | --- | +| Registered MFS accounts, Feb 2025 | 239.24 million (2,392.40 lakh) | [S1] | +| Active accounts, Feb 2025 | 87.15 million (derived: 36.4% of registered) | [S1] | +| Agents, Feb 2025 | 1,856,190 | [S1] | +| Providers | 13 | [S1] | +| Transactions, Feb 2025 | 671.3 million, worth Tk 164,726 crore (≈ Tk 1.65 trillion) | [S1] | +| Average transaction, Feb 2025 | ≈ Tk 2,454 (derived) | [S1] | +| Transactions, Oct 2025 | 678.63 million, worth Tk 1.58 trillion | [S2] | +| Average P2P send / cash-out / cash-in, Oct 2025 | Tk 3,552 / Tk 2,132 / Tk 2,898 (derived) | [S2] | +| Account holders by sex, Oct 2025 | 56% male, 44% female | [S2] | + +### Fraud and what happens to victims + +| Figure | Value | Source | +| --- | --- | --- | +| Payment fraud reported to Bangladesh Bank, 2025 | Tk 926.01 million across 81,423 cases; 10.7% recovered | [S3] | +| MFS share of it | Tk 813.26 million (88% of fraud value); only 8.7% recovered, Tk 742.56 million lost | [S3] | +| Personal MFS holders who were fraud victims | 6.3% (agents 17.0%, merchants 1.6%) | [S4] | +| Personal holders who lost money | 3.6% | [S4] | +| Personal losses reported | Tk 300 to Tk 83,000 | [S4] | +| How the fraud happened (multiple answers) | deceptive or false information 52.6%; phone call or SMS 42.1%; hacking 12.3% | [S4] | +| Victims who went to the police (case or GD) | 7.6% | [S4] | +| Holders with a problem who did not complain | 58.8%; the main reason was "no benefit" (65.8%) | [S4] | +| Holders aware of Bangladesh Bank's complaint centre (CIPC) | 6.2% | [S4] | +| Suspicious transaction/activity reports from MFS to BFIU | 451 (2022), 1,134 (2023), 1,107 (2024) | [S4] | +| Users who had experienced MFS fraud, 2021 survey of 9,279 | 9.3% ("1 in 10"); average loss above Tk 9,000; agents ≈ 13% | [S5] | +| Main fraud types in that survey | compromised PINs and impersonation | [S5] | + +The two surveys measure different things (any fraud in 2021 vs fraud victims +in the TIB sample, Nov 2023–May 2025), so they agree in size but should not be +read as a trend. + +### No public source found + +Per-provider scam loss rates; the share of scam money that is cashed out +within minutes; how long mule wallets stay active; the split of scams by +district; false-positive tolerance of Bangladeshi customers. These are what +the validation kit (docs/validation/) is designed to collect. + +## 2. Stakeholder map + +| Stakeholder | What they need | What FraudLens gives them | +| --- | --- | --- | +| **MFS customers** (239M accounts; 44% women) | Not to lose money; not to be blocked on a normal payment | Warn / step-up / hold before money leaves, in plain language; never a refused payment (DECISION_POLICY.md §2) | +| **MFS providers** (13; bKash, Nagad, Rocket, upay and others) | Lower fraud loss and complaint volume; regulatory compliance; low friction | Real-time scoring with reasons, analyst queue, mule network view, decision audit trail | +| **Agents** (1.86M; 17% have been fraud victims [S4]) | Protection from being used for cash-out; commission income | Agent risk score separating colluding/farming agents from transit hubs | +| **Bangladesh Bank** (PSD, FICSD, CIPC) | Providers that monitor patterns and resolve complaints in time | Monitoring that §10.1(ii) of the 2022 regulations asks for; case records kept and exportable | +| **BFIU** | Better suspicious-transaction reports | Case pages and network evidence that can back an STR | +| **Police / CID** | Evidence that follows the money | Mule chains and shared handsets across cases | + +## 3. Regulatory fit + +- **Bangladesh MFS Regulations 2022** (issued 15 Feb 2022) [S6]: + - §10.1(ii) — providers must monitor transaction patterns to identify + unauthorised or suspicious activity. FraudLens is that monitoring, in real + time. + - §15 — providers must run awareness programs "to combat against fraud and + forgery". The warn screen is awareness at the moment it matters. + - §17.3 — a 24-hour call centre and each dispute resolved within 10 working + days. The hold tier puts a human review on a 30-minute target. + - §17.4 — customers may complain to CIPC; §18.1 — records kept at least 6 + years. Every decision is logged with its model, policy version and reasons. +- **Bangladesh Bank, November 2025** — ordered the 13 MFS operators to deploy + "artificial intelligence–based systems to detect and flag illegal + transactions in real time" (issued about online gambling) [S7]. +- **TIB recommendations, May 2025** [S4] — an AI policy for suspicious + transaction reporting (#3), a central blacklist database (#9), a toll-free + CIPC line and a standard complaint process (#12–13). TIB also notes the + absence of proper use of AI in the sector. +- **The UK comparison** — since 7 October 2024 UK payment firms must reimburse + authorised-push-payment scam victims up to £85,000, split 50:50 between the + sending and receiving firm [S8]. In the first year 88% of money lost was + reimbursed (66% the year before) and 84% of claims were settled within 5 + business days [S9]. Bangladesh has no equivalent: 8.7% of MFS fraud value was + recovered in 2025 [S3]. If liability ever shifts to providers, catching the + payment before it leaves is far cheaper than paying it back. + +## 4. Pitch openers + +Each line is backed by the sources above. + +1. "In 2025, Bangladesh Bank recorded Tk 81 crore stolen through mobile + financial services — and only 8.7% of it came back." [S3] +2. "One in sixteen MFS users has been a fraud victim, and only 7.6% of them + ever went to the police." [S4] +3. "239 million MFS accounts move about Tk 1.6 trillion a month. Fraud + prevention today starts after the money is gone." [S1][S2][S3] +4. "Most MFS fraud is not hacking — 52.6% is deception. The customer types + their own PIN. Only a check before the payment can stop it." [S4] +5. "The UK now returns 88% of scam losses to victims. Bangladesh returns 8.7%. + Bangladesh Bank has already asked operators for real-time AI detection; this + is what that looks like." [S3][S7][S9] + +## Sources + +All accessed 7 October 2026. + +- **[S1]** Bangladesh Bank, *Mobile Financial Services (MFS) comparative + summary statement* (January and February 2025). + https://www.bb.org.bd/en/index.php/financialactivity/mfsdata +- **[S2]** The Financial Express, "MFS transactions maintain rising trend in + Oct '25", 3 January 2026, citing Bangladesh Bank. + https://thefinancialexpress.com.bd/trade/mfs-transactions-maintain-rising-trend-in-oct-25 +- **[S3]** The Financial Express, "Fraudsters gobble up Tk 926m in 2025", + 16 June 2026, citing Bangladesh Bank. + https://thefinancialexpress.com.bd/trade/fraudsters-gobble-up-tk-926m-in-2025 +- **[S4]** Transparency International Bangladesh, *Governance Challenges in + the Mobile Financial Services Sector and Way Forward* (executive summary), + 27 May 2025. Survey of 1,784 personal account holders, 664 agents and 429 + merchants in 32 districts, November 2023–May 2025. + https://www.ti-bangladesh.org/images/2025/report/mfs/Executive-Summary-Mobile-Financial-Services-Sector-En.pdf +- **[S5]** Policy Research Institute survey (Aug–Sep 2021, 9,279 respondents), + reported in Dhaka Tribune, 30 March 2022. + https://www.dhakatribune.com/business/266888/1-out-of-every-10-are-victims-of-mfs-fraud + and The Business Standard, https://www.tbsnews.net/node/394218 +- **[S6]** Bangladesh Bank, *Bangladesh Mobile Financial Services (MFS) + Regulations, 2022*. + https://www.bb.org.bd/aboutus/regulationguideline/psd/mfs_regulations_2022.pdf +- **[S7]** Yogonet, "Bangladesh Bank orders mobile financial service providers + to curb online gambling transactions", 6 November 2025. + https://www.yogonet.com/international/news/2025/11/06/116204-bangladesh-bank-orders-mobile-financial-service-providers-to-curb-online-gambling-transactions +- **[S8]** Payment Systems Regulator, *APP scams*, + https://www.psr.org.uk/our-work/app-scams/ ; DLA Piper, "PSR confirms APP + fraud reimbursement level to be reduced", 27 September 2024. + https://dlapiper.com/en-th/insights/publications/2024/09/psr-confirms-app-fraud-reimbursement-level-to-be-reduced +- **[S9]** Payment Systems Regulator, "One year on: impact of APP + reimbursement on victims", 8 October 2025. + https://www.psr.org.uk/news-and-updates/latest-news/news/one-year-on-impact-of-app-reimbursement-on-victims/ +- **[S10]** Dhaka Tribune, "bKash raises daily, monthly account transaction + limits", 29 March 2025 (Bangladesh Bank PSD notice of 27 March 2025). + https://www.dhakatribune.com/business/377554/bkash-raises-daily-monthly-account-transaction diff --git a/docs/BUSINESS_CASE.md b/docs/BUSINESS_CASE.md new file mode 100644 index 0000000..05e3414 --- /dev/null +++ b/docs/BUSINESS_CASE.md @@ -0,0 +1,185 @@ +# Business case — what the policy is worth in taka + +The model card and the decision policy report recall, precision and alerts a day. +This document turns those into a monthly profit and loss for a provider: money +kept from scammers, what the friction and the review team cost, and what is left. +It answers the three questions a CFO and a CRO ask first: is it worth running, +how many people does it need, and which of our own numbers would change the answer? + +**These are estimates on synthetic data.** The model rates come from the +back-test (model v4, policy v1 thresholds, 134,545 scored payments over 25 test +days the model never trained on). Everything that turns a rate into taka is an +assumption, listed in §2 with its source or reasoning. The simulator's fraud is +far denser than real traffic, so the scam money is rescaled to an assumed real +rate rather than taken from the simulation. A provider would replace the +assumptions with its own figures before relying on any number here. + +Code: `backend/src/fraudlens/decision/business.py`. API: +`GET /v1/model/business` (defaults) and `POST /v1/model/business` (any assumption +overridden). Console: the impact simulator, under "Business case". + +``` +cd backend && uv run python -m fraudlens.decision.business # the table below +uv run python -m fraudlens.decision.business --set monthly_payments=300000000 +``` + +## 1. How a threshold becomes taka + +The impact simulator's slider moves the warn threshold. Step-up and hold keep the +thresholds the policy gives them, or follow the slider once it passes them. For +each of the 40 thresholds in the sweep, and exactly at the policy in force: + +| Line | How it is computed | +| --- | --- | +| Scam money at risk | volume × average payment × scam rate | +| Kept from scammers | scam money alerted in each tier's band × the share that tier actually stops (warn, step-up, hold), plus mule cash-outs held after the victim's payment × the hold share. A warning shown to a mule stops nothing, so only holds count there | +| Friction | honest payments interrupted × abandonment rate × cost of an abandoned payment, plus every interruption × support contacts per tier × cost of a contact | +| Review | every held payment × analyst minutes; analysts = the larger of that workload and the round-the-clock floor, rounded up to whole people | +| Operating cost | friction + analysts + running the platform | +| Net benefit | kept × (1 + reputational cost per taka) − operating cost. Customers and provider together: every taka a customer keeps counts | +| Net benefit to the provider | kept × (reimbursement share + reputational cost per taka) − operating cost. Only the provider's own money | + +Rates per payment come from the back-test and are multiplied by the provider's +volume. The simulator's scams take ৳26.38 per scored payment (about 93 basis +points of payment value). At the assumed real rate of 2 basis points that is +scaled by 0.0212: scam money and true alerts shrink, false alerts do not, because +honest customers look the same whatever the scam rate. + +## 2. Assumptions + +| Assumption | Value | Range tried | Source or reasoning | +| --- | --- | --- | --- | +| Send-money and cash-out payments a month | 20 million | 10–40 million | Order of magnitude for a mid-sized provider (upay scale). Bangladesh Bank's monthly MFS statistics put the industry in the hundreds of millions of transactions a month; bKash scale is roughly 300 million scored payments. Replace with the provider's MIS figure | +| Average payment | ৳2,800 | ৳1,500–4,000 | The simulator's scored payments average ৳2,830 (send-money ৳2,247, cash-out ৳3,990) | +| Scam losses, share of payment value | 2 bp | 0.5–8 bp | Assumed. UK push-payment scam losses are about 1–2 bp of Faster Payments value; MFS in Bangladesh is assumed more exposed. No public Bangladeshi figure exists | +| Scam money stopped by a warning | 25% | 10–50% | Assumed: a coached victim often clicks through | +| … by a step-up check | 50% | 30–75% | Assumed: re-authentication stops most takeovers; the 30-minute cooling-off breaks a scam's urgency | +| … by a hold | 90% | 75–98% | Assumed: the money stays while an analyst calls; some victims insist | +| Honest payments abandoned after an interruption | 5% | 1–15% | Assumed | +| Cost of one abandoned payment | ৳25 | ৳10–60 | Lost fee and goodwill: cash-out pays about 1.85%, app send-money ৳0–5; many are retried | +| Support contacts per warning / step-up / hold | 0.02 / 0.10 / 0.25 | 0–0.05 / 0.05–0.25 / 0.10–0.50 | Assumed | +| Cost of one support contact | ৳50 | ৳25–100 | About six minutes of a contact-centre agent at ৳35,000 a month loaded, plus telephony | +| Analyst minutes per held payment | 8 | 4–20 | The impact page's default | +| Analyst cost a month, loaded | ৳60,000 | ৳40,000–100,000 | About ৳45,000 salary in Dhaka plus a third for benefits, seat, supervision | +| Hours of case work per analyst a month | 132 | 110–150 | Six hours of case work a shift, 22 shifts | +| Minimum analysts | 5 | 3–10 | Holds have a 30-minute target at any hour: one seat staffed 24x7 | +| Running the platform a month | ৳5 lakh | ৳2.5–15 lakh | Assumed: API and stream nodes, Postgres, Redis, part of an engineer | +| Share of scam losses the provider reimburses | 25% | 0–100% | Assumed. No mandatory rule in Bangladesh; the UK has required 100% since October 2024 | +| Reputational cost per taka a customer loses | ৳0.25 | ৳0–1 | Assumed, and the hardest to defend; shown separately in §4 | + +## 3. Results at upay scale (20 million payments a month) + +At the policy in force (warn at 1.49% of payments, measured exactly, not from the +nearest sweep point): + +| | A month | +| --- | --- | +| Scam money at risk | ৳1.12 crore | +| Kept from scammers | **৳88.6 lakh (79.1%)** — hold ৳73.4 lakh, mule cash-outs held ৳10.3 lakh, step-up ৳2.9 lakh, warn ৳2.0 lakh | +| Still lost | ৳23.4 lakh | +| Friction | ৳4.6 lakh — 5,164 abandoned honest payments (৳1.3 lakh), 6,664 support contacts (৳3.3 lakh) | +| Holds to review | 11,544 → 1,539 hours → 11.7 FTE of work → **12 analysts** (৳7.2 lakh) | +| Platform | ৳5.0 lakh | +| Operating cost | ৳16.8 lakh | +| **Net benefit** | **৳94.0 lakh a month (about ৳11.3 crore a year)** | +| Net benefit to the provider alone | ৳27.5 lakh a month (about ৳3.3 crore a year) | +| Kept per ৳1 of operating cost | **৳5.27** | +| Honest customers interrupted | **51.6 per 10,000 payments** (103,277 a month) | +| Break-even scam rate | 0.27 bp (0.69 bp for the provider alone) | + +Across the sweep (selected points): + +| Payments interrupted | Kept from scammers | Operating cost | Net benefit | Provider net | Analysts | Honest per 10,000 | Kept per ৳1 | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 0.05% | ৳9.3 lakh | ৳8.0 lakh | ৳3.5 lakh | −৳3.4 lakh | 5 | 0.0 | 1.15 | +| 0.53% | ৳58.1 lakh | ৳8.3 lakh | ৳64.3 lakh | ৳20.7 lakh | 5 | 0.1 | 7.00 | +| 0.96% | ৳86.1 lakh | ৳14.9 lakh | ৳92.8 lakh | ৳28.2 lakh | 12 | 13.2 | 5.79 | +| **1.21% (best)** | ৳88.0 lakh | ৳15.8 lakh | **৳94.2 lakh** | ৳28.2 lakh | 12 | 30.0 | 5.56 | +| **1.49% (policy)** | ৳88.6 lakh | ৳16.8 lakh | **৳94.0 lakh** | ৳27.5 lakh | 12 | 51.6 | 5.27 | +| 2.46% | ৳90.3 lakh | ৳20.8 lakh | ৳92.0 lakh | ৳24.3 lakh | 12 | 140.5 | 4.33 | +| 5.00% | ৳91.4 lakh | ৳31.9 lakh | ৳82.3 lakh | ৳13.7 lakh | 12 | 387.6 | 2.86 | + +What this says: + +- **The thresholds in force sit at the top of the curve.** The best point in the + sweep nets ৳22,600 a month more (0.2%) by interrupting 1.21% of payments instead + of 1.49%. Loosening further buys little: going from 1.49% to 5% keeps another + ৳2.8 lakh and costs ৳15 lakh more in friction. +- **Holds do the work.** 94% of the money kept comes from the hold tier and the + mule cash-outs it catches; warnings add 2%. That is why the assumed hold + effectiveness matters more than the warning's. +- **At real scam rates most interruptions are honest customers.** In the + simulator 65% of warn-level alerts are fraud; rescaled to 2 bp, 96% of the + 107,406 interruptions a month are honest payments, and about 8,500 of the + 11,544 holds are. The friction cost and the 12 analysts are sized for that. + The cost per honest customer is low (most see one warning), but a CRO should + see the count: about 1 in 200 payments. + +### At bKash scale (300 million payments a month) + +Same assumptions, volume ×15: ৳16.8 crore at risk, **৳13.3 crore kept**, operating +cost ৳1.79 crore (175 analysts, ৳1.05 crore; friction ৳69 lakh), **net benefit +৳14.8 crore a month**, ৳4.85 crore to the provider alone, ৳7.41 kept per ৳1. +The 175 analysts are a real constraint: 173,000 holds a month need a review +operation of that size, or a stricter hold threshold. + +## 4. Sensitivity + +Net benefit of the policy in force at upay scale (৳94.0 lakh a month; ৳27.5 lakh +to the provider), with one assumption at a time moved to the ends of its range. +Largest swing first. + +| Assumption | Low → net benefit (provider) | High → net benefit (provider) | Swing | +| --- | --- | --- | --- | +| Scam losses, share of payment value | 0.5 bp → ৳12.4 lakh (−৳4.2 lakh) | 8 bp → ৳4.20 crore (৳1.54 crore) | ৳4.08 crore | +| Payments a month | 10 M → ৳44.5 lakh (৳11.2 lakh) | 40 M → ৳1.93 crore (৳60.0 lakh) | ৳1.48 crore | +| Average payment | ৳1,500 → ৳43.3 lakh (৳7.7 lakh) | ৳4,000 → ৳1.41 crore (৳45.7 lakh) | ৳97.3 lakh | +| Reputational cost per taka lost | 0 → ৳71.8 lakh (৳5.3 lakh) | 1 → ৳1.60 crore (৳94.0 lakh) | ৳88.6 lakh | +| Scam money stopped by a hold | 75% → ৳76.5 lakh (৳20.5 lakh) | 98% → ৳1.03 crore (৳31.2 lakh) | ৳26.7 lakh | +| Analyst minutes per hold | 4 → ৳97.6 lakh (৳31.1 lakh) | 20 → ৳83.2 lakh (৳16.7 lakh) | ৳14.4 lakh | +| Running the platform | ৳2.5 lakh → ৳96.5 lakh | ৳15 lakh → ৳84.0 lakh | ৳12.5 lakh | +| Analyst cost a month | ৳40,000 → ৳96.4 lakh | ৳100,000 → ৳89.2 lakh | ৳7.2 lakh | +| Cost of one support contact | ৳25 → ৳95.6 lakh | ৳100 → ৳90.6 lakh | ৳5.0 lakh | +| Scam money stopped by a warning | 10% → ৳92.5 lakh | 50% → ৳96.4 lakh | ৳3.9 lakh | +| Honest payments abandoned | 1% → ৳95.0 lakh | 15% → ৳91.4 lakh | ৳3.6 lakh | +| Scam money stopped by a step-up | 30% → ৳92.5 lakh | 75% → ৳95.8 lakh | ৳3.3 lakh | +| Cost of an abandoned payment | ৳10 → ৳94.7 lakh | ৳60 → ৳92.1 lakh | ৳2.6 lakh | +| Support contacts per step-up | 0.05 → ৳94.5 lakh | 0.25 → ৳92.2 lakh | ৳2.3 lakh | +| Support contacts per hold | 0.10 → ৳94.8 lakh | 0.50 → ৳92.5 lakh | ৳2.3 lakh | +| Support contacts per warning | 0 → ৳94.7 lakh | 0.05 → ৳92.9 lakh | ৳1.8 lakh | +| Hours of case work per analyst | 110 → ৳92.8 lakh | 150 → ৳94.6 lakh | ৳1.8 lakh | +| Share reimbursed by the provider | 0% → ৳94.0 lakh (৳5.3 lakh) | 100% → ৳94.0 lakh (৳94.0 lakh) | 0 (provider ৳88.6 lakh) | +| Minimum analysts | 3 → ৳94.0 lakh | 10 → ৳94.0 lakh | 0 (workload needs 12 anyway) | + +Reading it: + +- **The answer depends on the scam rate and the volume, not on the cost lines.** + Every friction and staffing assumption together moves the net benefit by less + than the plausible range of the scam rate alone. The case stays positive for + customers and provider together down to 0.27 bp of scam losses, and for the + provider's own money down to 0.69 bp. +- **For the provider's own P&L, reimbursement and reputation decide it.** With no + reimbursement and no reputational cost the provider still nets ৳5.3 lakh at the + default scam rate; at 0.5 bp it would lose ৳4.2 lakh a month while its + customers kept ৳22.2 lakh (`--set scam_loss_bps=0.5`). Whether that is worth + paying for is a policy choice, and the table makes it visible. +- **The first number to replace is the scam rate.** It is the largest swing and + the least known; a provider's complaint and reimbursement records give it + directly. + +## 5. What this does not show + +- **The model rates are synthetic.** Recall and false-alert rate come from a + simulator ([DATA_ASSUMPTIONS.md](DATA_ASSUMPTIONS.md)). Real scams the model has + not seen will be caught less often; the held-out scam type is caught at 70.4% + against 86.9% overall (MODEL_CARD). +- **Rescaling assumes the scam mix and the alert ranking stay as they are** when + scams are rarer. Real precision would come from the provider's own back-test. +- **Tier effectiveness is assumed.** Nothing here measures how many warned victims + stop; a live A/B of warning texts would. +- **Warnings and step-ups cost no analyst time**, as the policy specifies; only + holds go to a person. A provider that reviewed step-ups would need more people. +- **The platform cost is flat.** At bKash scale it would be higher, though still + small next to the review team. +- **No churn model.** Abandonment is priced per payment; customers who leave + after a false hold are not counted beyond the reputational line. diff --git a/docs/CONSORTIUM.md b/docs/CONSORTIUM.md new file mode 100644 index 0000000..f719051 --- /dev/null +++ b/docs/CONSORTIUM.md @@ -0,0 +1,175 @@ +# Mule-intelligence consortium + +A mule that is reported at one provider usually still has a wallet at another one. +Today each provider learns this on its own, one victim at a time. The consortium lets +the providers warn each other about mule wallets **without any of them handing over a +customer's number, handset or identity**. + +Everything here is a simulation on the FraudLens dataset. The four providers, their +market shares and the share of people with a second wallet on the same SIM are +*assumptions*, labelled "(simulated)" in the console. The protocol is real and +running code: `backend/src/fraudlens/consortium/`, Python standard library only. + +| Piece | Where | +|---|---| +| Crypto: OPRF, Schnorr signatures, Bloom filter | `consortium/crypto.py` | +| Bundle, listing, ledger, lookup, audit chain | `consortium/protocol.py` | +| Hub (token service, relay, disputes) and Member | `consortium/hub.py` | +| Four-provider market and the experiment | `consortium/simulate.py` → `artifacts/consortium/report.json` | +| Demo state for the console | `consortium/demo.py` | +| API | `GET /v1/consortium`, `/matches`, `/audit`; `POST /lookup`, `/disputes`, `/disputes/{id}/resolve` | +| Console | **Network → Mule consortium** (also linked from the rings page) | +| Tests | `backend/tests/test_consortium.py` | + +``` +uv run python -m fraudlens.consortium.simulate --quick # one seed, ~10 min +uv run python -m fraudlens.consortium.simulate # 5 seeds x 3 SIM-sharing rates +``` + +Nothing else changes: the dataset, features, served model and headline numbers are +untouched; the routes answer 404 (`consortium_not_built`) until the simulation has run. + +## Protocol + +**Identifiers.** Two kinds: the wallet number (MSISDN, normalised to `8801XXXXXXXXX`) +and the handset id. Names, NIDs and balances are never part of the protocol. + +**Tokens (oblivious PRF).** A plain hash or HMAC of a phone number is reversible: the +Bangladeshi numbering plan has about 10⁹ numbers, an afternoon of hashing. So tokens +come from an OPRF held by the hub, in the 2048-bit MODP group of RFC 3526: + +1. The provider maps the identifier into the group, `h = H(kind, id)`, and blinds it + with a fresh random `r`: it sends `h · g^r`. +2. The hub raises it to its secret key `k` and returns it. It sees a random group element. +3. The provider removes the blind with the hub's public key, getting `h^k`, and hashes + that (with the key epoch and kind) to a 128-bit token. + +Same identifier, same token at every provider; the hub never learns the identifier; +no provider can make tokens offline, so it cannot brute-force partners' lists. Token +requests are per-member rate-limited (`DEFAULT_QUOTA`, 250,000 a day) and audited. +Providers cache their own customers' tokens for the key epoch, so a payment-time +lookup normally needs no hub round trip. + +**Bundles.** Once a day each provider publishes a signed bundle: + +- *confirmed* listings (victim-reported mules): token, kind, confidence, typology, + first-seen, listed-at, **expiry (180 days)** and a listing id; +- *suspected* wallets (its own mule-model alerts, **30-day expiry**) only inside a Bloom + filter at 10⁻⁶ false hits per lookup: partners can test a token, not list the set; +- the ids it has withdrawn. + +Bundles are Schnorr-signed by the issuing provider and themselves expire after two days, +so a feed that stops being refreshed stops counting. Importers check the signature, key +epoch, sequence number (no replay) and expiry, and reject their own bundle. + +**Lookup.** At each transfer the receiving provider tokenises the receiver's number and +handsets and checks them against partner bundles. The signal is +`1 − Π(1 − c_p)` over partners `p`, each counted once at its best confidence `c_p`. +A lookup is audited with counts and listing ids, never values. + +**Dispute and removal.** Any member can dispute a listing (for instance on its own +customer's appeal). The listing stops counting **for everyone at once**. Only the +listing member decides (it holds the evidence): *upheld* restores it, *withdrawn* removes +it for good. An unanswered dispute is decided **for the customer** after 5 days. + +**Audit.** The hub and every member keep a hash-chained log (each entry commits to the +previous hash): joins, token requests and refusals, shares, imports, feeds, lookups, +disputes. Tampering with any entry breaks the chain (`AuditChain.verify`). Console +lookups and disputes are also written to the platform's Postgres audit log. + +## Threat model + +| Threat | What stops it | Residual risk | +|---|---|---| +| **Reversing tokens** (dictionary over 01XXXXXXXXX) | OPRF: tokens need the hub key, one rate-limited, audited request at a time | A member can still test *its own* candidates at quota rate; quota and audit make bulk probing visible | +| **Hub learns identifiers** | Blinding: the hub sees `h·g^r`, uniformly random | The hub sees request *volumes* per member | +| **Linkage across epochs** | Tokens bind the key epoch; rotating `k` makes old tokens useless | Within an epoch a token is a stable pseudonym among members | +| **Listing enumeration of suspected wallets** | Suspected wallets are only in a Bloom filter | A member can test tokens it already holds | +| **Poisoning by a malicious member** (listing innocents) | Per-listing attribution via signatures; dispute → instant suspension; disputes-against counted per member; rule-based use of *confirmed* listings only | Measured below: it does hurt until disputes catch up. Governance must be able to suspend a member | +| **Forged or altered listings by the hub** | Schnorr signature per provider; hub only relays | — | +| **Stale or replayed feeds** | Sequence numbers, 2-day bundle expiry | — | +| **Hub key compromise** | Keep `k` in an HSM; rotate per quarter (`key_epoch`); members re-tokenise their own lists | Past bundles become linkable to whoever holds the old key and a candidate list | +| **Member signing-key compromise** | Hub re-registers the member's key; bundles under the old key rejected | Listings forged until rotation | +| **False listing harms a customer** | Expiry, dispute, default-for-customer after 5 days; a match is a signal to a reviewer, never an automatic freeze | — | + +The simulated keys in `artifacts/consortium/demo_keys.json` are all held by one process +for the demo. In a deployment nobody holds more than its own. + +## Governance and Bangladesh alignment + +- **Who runs the hub.** A neutral operator under Bangladesh Bank oversight (the Payment + Systems Department, alongside the NPSB switch), not one of the providers. +- **Legal basis.** Sharing is for fraud prevention under the MFS regulations and the + BFIU's anti-money-laundering framework, which already oblige providers to act on mule + accounts and report suspicious transactions. Tokens and Bloom filters keep it to the + minimum needed, in line with the data-minimisation and purpose-limitation principles + of Bangladesh's personal-data-protection framework. (Legal review is still needed; + this is the design intent, not legal advice.) +- **Membership.** Signed membership agreement; per-member quotas; a member whose + listings are repeatedly withdrawn after dispute is reviewed and can be suspended (the + hub counts `disputes_against` and `withdrawn_after_dispute`). +- **Customers.** A listing only ever raises a review or a step-up at the receiving + provider. Customers can appeal through their own provider, which raises a dispute. +- **Retention.** 180 days for confirmed, 30 for suspected, 2 for a bundle; audit logs + kept for the regulator. + +## Measured results + +From `artifacts/consortium/report.json` (full run: 5 seeds × 3 SIM-sharing rates, 552 s). +Test period 2026-04-06 → 2026-05-01. It contains 145 mule wallets and about 14,800 +innocent wallets. The served mule model and its threshold (0.1485) are unchanged in +every arm. Listing confidences are measured, not assumed: + +- *confirmed* = 0.893, the precision of the 261 wallets reported before the test period; +- *suspected* = 0.586, the precision of the 58 wallets the mule model alerted on val_b. + +The seed changes only which wallet shares a SIM with which. "Shared SIM" is the +simulated share of people who have a second wallet on the same SIM at another provider. + +**Main result, 25 % shared SIM.** Mean over 5 seeds, with the [min, max] range. + +| Arm | Recall | False alerts (FPR) | Mules found only via consortium | Detected before any victim paid | Victim transfers before detection | ৳ paid to mules never detected | Median hours ahead of report | +|---|---|---|---|---|---|---|---| +| Mule model only (served) | 0.648 | 40 (0.27 %) | — | 70 | 72 | 706,840 | 40.2 | +| + confirmed MSISDN listings (rule) | **0.693** [0.690, 0.697] | 40.8 (0.28 %) | **6.4** [6, 7] | **77.2** [76, 78] | 69.6 | **656,844** | 41.8 | +| + MSISDN, threshold tuned on val_b | 0.684 [0.648, 0.697] | 40.6 (0.27 %) | 5.2 [0, 7] | 76.0 | 69.6 | 662,156 | 41.7 | +| + handset listings too (rule) | 0.693 | 113.8 (0.77 %) | 6.4 | 77.2 | 69.6 | 656,844 | 41.8 | +| handset only / MSISDN+handset, tuned | 0.648 | 40 | 0 | 70 | 72 | 706,840 | 40.2 | + +**Effect of the shared-SIM rate.** Confirmed-MSISDN rule arm; the baseline is the same at every rate. + +| Shared SIM | Recall | False alerts | Mules found only via consortium | ৳ to mules never detected | +|---|---|---|---|---| +| 0 % (control) | 0.648 | 40 | 0 | 706,840 | +| 25 % | 0.693 | 40.8 | 6.4 | 656,844 | +| 50 % | **0.728** [0.710, 0.759] | 42.0 | **11.6** [9, 16] | **561,170** | + +The 0 % row is a control. When nobody has a second wallet on a shared SIM, MSISDN +listings match nothing, and the measured gain is exactly zero. + +How to read this: + +- **Keep fraud-team precision; raise recall.** Confirmed MSISDN listings add about 6 + mules at 25 % shared SIM (+4.4 recall points) and about 12 at 50 % (+8 points). That + costs at most 2 extra false alerts on roughly 14,800 innocent wallets. 7 more mules + (77 vs 70) are caught before any victim pays them. Money sent to mules that are never + caught falls by ৳50k at 25 % shared SIM and ৳146k at 50 % (−7 % and −21 %). +- **Handset listings are not worth it.** In every run they add 63–80 false alerts and + find no extra mules: innocents share handsets with mules. Keep them as a review hint. + They must never trigger an alert on their own. +- **Tuning on val_b is too timid.** val_b holds only 39 mules, so a threshold tuned + there often falls back to the model alone. Using confirmed listings as a rule, at + their measured precision, works better. +- **Poisoning is real.** In one test, one member (upay) lists 500 innocent numbers as + "confirmed". The rule arm's false alerts rise from 41 to 462 (FPR 3.1 %). Disputes were not simulated; in practice the damage lasts until + disputes suspend those listings. That is why the dispute route suspends a listing for + all members at once, and why the hub counts `disputes_against` for each member. +- **Cost.** One OPRF token takes 6.3 ms on the client and 2.8 ms at the hub. The run + makes 116,087 tokens, one per wallet identifier, cached for the key epoch. + +## Not done + +- Real PSI with cardinality hiding, and threshold OPRF across several hubs. +- Per-member weighting learned from dispute outcomes. +- Production key management (HSM) and network transport; the demo is one process. +- A hosted multi-party deployment: the API serves a saved simulated state. diff --git a/docs/DATA_ASSUMPTIONS.md b/docs/DATA_ASSUMPTIONS.md index eae9a6d..c0bbdf8 100644 --- a/docs/DATA_ASSUMPTIONS.md +++ b/docs/DATA_ASSUMPTIONS.md @@ -208,13 +208,17 @@ behaviour that was missing. But the test period is no longer strictly ## 10. Known limitations -- Rates, shares and amounts are informed guesses, not calibrated to real traffic. +- Rates, shares and amounts are informed guesses, not calibrated to real traffic + (the optional calibrated profile, §11, sets the few that have a public source). Absolute metrics will not transfer; the relative findings and the system design are what we claim. - Fraud prevalence is likely higher than in real traffic, to give enough positives to evaluate each typology. -- Fraud scripts are stylised. Real scammers adapt to controls; the simulator's do - not react to the model's decisions. +- Fraud scripts are stylised. Real scammers adapt to controls; the default + simulator's do not react to the model's decisions. The opt-in adversary mode + (`simulator/adversary.py`, MODEL_CARD §14) adds cells that shift between five + evasion tactics according to what got through. Its tactics, their parameters + and its learning rule are assumptions too. - No merchant fraud, no loan or credit fraud, no SIM-swap telemetry, no customer demographics beyond segment and district. - Reports are the only source of confirmed labels and arrive for about half of @@ -224,3 +228,51 @@ behaviour that was missing. But the test period is no longer strictly review simulator (`make review`) closes cases with the simulator's ground truth, so the labels the feedback loop learns from are more accurate and more complete than real verdicts would be (MODEL_CARD §13). + +## 11. Calibration against Bangladesh sources + +Most of the parameters above have no public source. A few do, and the +**calibrated** profile (`SimConfig.calibrated()`) sets those from public +Bangladesh figures and leaves everything else alone. The **default** profile is +unchanged: every published model, policy and metric still comes from it, and a +test checks that `with_profile("default")` returns the same config. + +``` +cd backend +uv run python -m fraudlens.simulator.generate --profile calibrated # writes data/full_calibrated +FRAUDLENS_SIM_PROFILE=calibrated uv run python -m fraudlens.simulator.generate +``` + +Sources are listed in full, with access dates, in BANGLADESH_CONTEXT.md. + +| Parameter | Default | Calibrated | Sourced value | Source | +| --- | --- | --- | --- | --- | +| `txn_cap` (largest single wallet transaction) | ৳25,000 | ৳30,000 | Bangladesh Bank publishes daily limits, not a per-transaction one: cash-out ৳30,000/day, send money ৳50,000/day (from 27 Mar 2025). We use the smallest. | [S10] | +| `send_median` (send-money amount) | ৳1,500 → mean ৳2,425 | ৳2,850 → mean ৳3,520 | Average P2P transfer ৳3,552 (Oct 2025, derived from 134.26M txns worth ৳476.93bn) | [S2] | +| `cash_out_median` (spontaneous cash-out) | ৳3,000 → mean ৳4,336 | ৳1,400 → mean ৳4,337 | Average cash-out ৳2,132 (Oct 2025, derived). **Not matched**: most simulated cash-outs are balance-driven withdrawals after salary and remittance, which this parameter does not set. | [S2] | +| `report_rate` (share of victim transfers reported) | 0.50 (assumed) | 0.412 | 41.2% of personal holders with a problem complained (TIB). The denominator is all problems, not only fraud, so this is an approximation. | [S4] | +| Account-takeover share of base-typology cases (`ato_keep_share`) | 24.8% (keep 1.0) | 12.5% (keep 0.5; the rest become impersonation) | Hacking 12.3% of fraud reports; deceptive information 52.6% | [S4] | +| Cash-in amount | mean ৳2,978 | unchanged | Average cash-in ৳2,898 (Oct 2025, derived) — already close | [S2] | +| Overall mean transaction | ৳2,456 | ৳2,957 | ৳2,454 (Feb 2025, derived from ৳164,726 crore / 671.3M). The default matched without being tuned to it; the calibrated profile moves away because of the higher send amounts. | [S1] | +| Mean loss per case | ৳8,982 | ৳11,157 | "Over ৳9,000" average loss (2021 survey); TIB range ৳300–83,000 | [S5], [S4] | + +**Effect on the data (full world, seed 7).** 1,207 cases instead of 1,056; the +typology mix moves from 21% to 10.5% account takeover and from 19% to 31% +impersonation. The random stream of the default profile is untouched (the +extra draw for takeovers only happens when `ato_keep_share < 1`). + +**Not calibrated, and why.** + +- *Transaction mix by type.* Bangladesh Bank's Oct 2025 counts give P2P 19.8%, + cash-in 21.0%, cash-out 25.7%; the simulator has 29%, 20% and 15%. The mix + comes out of many behaviours (salary, remittance, agent farming) rather than + one parameter, and re-weighting it is a larger change than this profile makes. +- *Fraud prevalence.* No public source gives a per-transaction scam rate; the + simulator's rate stays deliberately high (§10). +- *No public source found* for district weights, segment shares, mule dwell + times and lifetimes, report delay, colluding-agent share or cell structure. + The validation kit (docs/validation/) is designed to collect several of these + from customers and analysts. + +The calibrated profile has not been used to retrain or re-evaluate the model; +doing so, and reporting the change, is the next step. diff --git a/docs/DECISION_POLICY.md b/docs/DECISION_POLICY.md index a8f555e..5da6d18 100644 --- a/docs/DECISION_POLICY.md +++ b/docs/DECISION_POLICY.md @@ -279,8 +279,8 @@ check. ## 9. Changing the policy -1. Copy `policies/v2.yaml` to `v3.yaml`, set `version: v3`, edit. -2. `uv run python -m fraudlens.decision.evaluate --policy v3` reports what the +1. Copy the newest file in `policies/` (now `v3.yaml`) to the next version, set `version:`, edit. +2. `uv run python -m fraudlens.decision.evaluate --policy v4` reports what the change does to every tier before anything is served. 3. A file that is malformed, has an unknown field, removes human review from holds, or declares a different version than its file name is refused at load. @@ -289,5 +289,81 @@ check. ``` make policy # writes policy_report.json and similar_cases.npz next to the model -make test # 203 tests; 79 cover this layer +make mitigation # fits and measures the v3 segments; writes fairness_mitigation.json +make test # 246 tests (52 need Postgres and skip without it); 19 cover the segments ``` + +## 11. Young wallets: segment thresholds (policy v3) + +v2 interrupts honest payments involving a wallet under 30 days old far more than +others ([MODEL_CARD §13](MODEL_CARD.md)). v3 is v2 plus a `segments:` block; v1 and +v2 are unchanged, v2 stays the default, and `FRAUDLENS_POLICY_VERSION=v3` serves v3. + +**What a segment is.** A set of conditions (`r_age_days < 30` on a transfer; +`s_age_days < 30` on any payment) and a `scale` per tier. Its cut-offs are the +model's times the scale, so they follow whichever model version is resolved, +including a shadow model. The first matching segment decides the model's tier; +the rules then run exactly as for any payment, so R01/R02 still hold confirmed +fraud and the mule rules still raise. Each decision records the segment and the +cut-offs used (`segment` in the decision detail), and the 0–100 display score is +anchored to those cut-offs. + +**Guards, enforced when the thresholds are resolved.** + +- A segment's warn cut-off is capped at the policy's hold cut-off: a payment the + model would hold for anyone is at least warned in every segment. +- A segment's hold cut-off stays below 1 (it keeps at least 1/scale of the room + between the policy's hold cut-off and 1), so every segment can still be held for + a person; the hold tier keeps `human_review: true` as before. +- Cut-offs out of order are refused at load, as for the policy's own. + +**The softer tier.** A hold scale above 1 means a payment between the policy's +hold cut-off and the segment's gets step-up (re-authenticate and wait 30 minutes) +instead of money held for an analyst. Step-up rather than a warning, because a +young sender may be a takeover victim and step-up still stops someone who does +not hold the credentials. + +**How the scales were chosen** (`fraudlens.decision.mitigation`, on val_a + val_b +only, 110,090 payments). Warn scales 1–250 and hold scales 1–4 per segment, every +combination (6,400). Objective: minimise the larger of the two young-wallet +false-alert ratios, each against honest payments with no young wallet (a rate the +segments cannot move; against the overall rate, helping one group raises the +other's ratio). Subject to: at most 0.5 pp of victim transfers alerted lost, 0.5 +pp of victim money alerted, 1 pp of victim transfers held, and the guard. Ratios +within 0.1 count as equal; ties go to the smaller sum of the two, the smaller +false-hold ratio, then the smaller change. 2,420 combinations met the +constraints. Chosen: + +| Segment | Warn | Step-up | Hold | Cut-offs on model v4 (warn / step-up / hold) | +| --- | --- | --- | --- | --- | +| S01 receiving wallet under 30 days | ×25 | ×6.03 | ×3 | 0.0496 / 0.0496 / 0.1494 | +| S02 sender under 30 days | ×16 | ×3.86 | ×1 | 0.0318 / 0.0318 / 0.0498 | + +For both segments the warn and step-up cut-offs coincide, so below the hold a +young wallet's payment gets step-up or nothing. On validation the choice costs no +victim recall at warn (97.6% before and after; held 96.4% to 95.8%). + +**Measured on test** (`fairness_mitigation.json`; the full table with 95% +bootstrap intervals is in MODEL_CARD §13): + +| | v2 | v3 | +| --- | --- | --- | +| Young senders, times the overall false-alert rate | 5.90 | 1.14 | +| Young receiving wallets, times the overall rate | 16.17 | 7.03 | +| Young receiving wallets, honest payments held | 1.50% | 0.70% | +| Victim transfers alerted / held | 81.73% / 66.47% | 81.58% / 66.19% | +| Victims' money alerted | 85.19% | 84.76% | +| Alerts a day, precision at warn | 81.0, 64.7% | 71.7, 73.1% | + +The cost on test is one victim transfer no longer alerted and two moved from hold +to step-up (৳15,300 of victims' money no longer alerted), against 232 honest +payments no longer interrupted and 18 honest holds softened. The report also +checks that the engine's decisions equal the fit's arithmetic for every test row +and that the policy file carries the chosen scales. + +**What it does not do.** The gap for young receiving wallets is narrowed, not +closed: even with no guard (warn ×250) it stays about 6× the no-young-wallet rate, +because the rest comes from rules and from honest payments the model scores at +hold level. Young receiving wallets on USSD remain the worst group (16× the +overall rate in rural areas). Recommendation: serve v3 once a person has accepted +the cost of one victim transfer in 695; watch young-wallet USSD transfers. diff --git a/docs/FRAUD_TAXONOMY.md b/docs/FRAUD_TAXONOMY.md index 73f490f..d07523c 100644 --- a/docs/FRAUD_TAXONOMY.md +++ b/docs/FRAUD_TAXONOMY.md @@ -61,7 +61,7 @@ What each detector contributes per category: | Where | What | | --- | --- | -| `POST /v1/customer/message-check` | `{wallet_id, text}` → `level` (`none`, `caution`, `high`), `risk`, the categories it looks like, the cues that matched, each link and what is wrong with it, and the fixed advice in English and Bangla. 20 a minute per wallet. | +| `POST /v1/customer/message-check` | `{wallet_id, text}` → `level` (`none`, `caution`, `high`), `risk`, the categories it looks like, the cues that matched with the exact phrases, the words that weighed most (`highlights`), each link and what is wrong with it, and the fixed advice in English and Bangla. 20 a minute per wallet. A flagged check makes a later payment to the number or amount it named a warning (§4). | | `POST /v1/customer/payment-verify` | `{wallet_id, txn_id?, amount?, message?}` → `verified`, `mismatch` or `not_found`, the checks behind it, and a fixed message. The ID and amount are read from the pasted message when not given. 10 a minute per wallet. | | `GET /v1/decisions/{txn_id}` | `fraud_categories`: the categories an alert belongs to and what that rests on (`scenario`, `similar_cases`, `mule_score`). | | `GET /v1/cases/{id}` | each customer report carries `fraud_categories`, from the category the customer picked or, for "something else", from the description. | @@ -85,93 +85,193 @@ Rules that hold for all of them: ## 4. The message classifier -**Corpus.** No real customer message was used. `intel/corpus.yaml` holds 300 -templates in 51 scripts ("families"), 36 of them scams, in English, Bangla and -Banglish (Bangla in Latin letters, how most SMS and chat is written). Slots are -filled with generated names, amounts and numbers to make 7,572 messages. The -harmless side is deliberately hard: real OTP notices that say "never share this -code", cash-in receipts, delivery messages, friends asking for money. +**Two sources of text.** -**Splits.** Test messages come from *templates* the model never saw, so the test -measures new wording, not memorised sentences. Eleven whole scripts (8 scam, 3 -harmless) are kept out of training entirely as the `unseen` split: customs parcel, -bank card, eSIM porting, overpayment, P2P mule, rental and ticket, charity visa, -utility-bill link. +- *Our own scripts.* `intel/corpus.yaml` holds 413 templates in 60 scripts + ("families"), 42 of them scams, in Bangla script (`bn`), Banglish (`bl`, Bangla in + Latin letters, how most SMS and chat is written), English (`en`) and code-mixed + Bangla with English words (`mx`, "OTP টা বলুন"). Every line was written for this + project; none is a real message. Slots are filled with generated names, amounts + and numbers: 10,566 messages (bl 3,386, bn 2,182, en 3,612, mx 1,386). The + harmless side is deliberately hard: real OTP notices that say "never share this + code", cash-in receipts, delivery messages with tracking links, a parent asking + for money on a mobile wallet. +- *A public corpus.* "Bengali SMS Smishing Dataset", Hugging Face + `shariul-islam/bengali-sms-smishing-dataset`, **MIT licence**, 7,005 SMS in the + same four varieties, labelled `smish`, `promo` or `normal` by its authors. It is + fetched at a pinned revision (`9d7c131c…`) and checked against fixed SHA-256 sums + (`intel/external.py`); nothing from it is committed. `smish` counts as a scam, + the other two as harmless. It has no category labels, so it trains the + scam-or-not output only. Its own split is kept: 4,903 train, 701 validation, + 1,401 test, and the test part is never trained on. Copies of the same dataset + re-uploaded under other accounts were not used; the original is. -**Model.** One-vs-rest logistic regression over character n-grams (2–5, TF-IDF) +**Typologies.** Every scam script is tagged with the shape customers meet most +often: fake agent or helpline, prize or lottery, wrong-number send-back, job or +investment, fake government aid, OTP or PIN phishing, or `other`. Each of the six +has trained scripts *and* at least one held-out script. + +**Three test sets.** + +| | What it measures | Messages | +| --- | --- | --- | +| `test` | new wording of scripts it was trained on (templates never seen) | 1,591 | +| `unseen` | 17 whole scripts never trained on, at least one per typology (14 scam, 3 harmless) | 1,950 | +| `external` | the public corpus's test split: text written by other people | 1,401 | + +**Model.** Logistic regression over character n-grams (2–5, TF-IDF, word-bounded) plus 27 hand-written cue patterns (asks for a secret, threat, urgency, pay first, -windfall, …) and their pairwise products. Nine outputs: scam or not, and the eight -categories. Text is normalised first (Bangla digits to ASCII, links to one token, -digits to `0`), so the model learns shapes, not numbers. Thresholds are set on the -validation split: `caution` at ≤ 3% of harmless messages flagged, `high` at ≤ 0.5%. -Training takes about four seconds (`make intel`). +windfall, …) and their pairwise products. Nine outputs, one regression each: scam +or not, and the eight categories; rows without a category label (the public +corpus) are left out of the category outputs. Text is normalised first (Bangla +digits to ASCII, links to one token, digits to `0`). Thresholds are set on the +harmless validation messages of each source separately and the stricter one is +kept: `caution` flags ≤ 3% of them, `high` ≤ 0.5%. Training takes about nine seconds +(`make intel`); without network access it trains on our scripts alone, which is +the previous recipe. -**Why it says so.** The reasons returned are the cues that matched, each with a -fixed English and Bangla label. +**Why this model.** `make intel-compare` trains five candidates on the same +splits, each on our scripts alone and on our scripts plus the public corpus +(`+ext`), with the same threshold rule. Macro-F1 is over scam and harmless at the +`caution` threshold; latency is one message at a time on a laptop CPU (Apple +silicon, 200 calls). -## 5. Measured +| Candidate | `test` macro-F1 | `unseen` macro-F1 | `unseen` recall / false alarms | `external` AUC | `external` macro-F1 | p50 / p95 ms | +| --- | --- | --- | --- | --- | --- | --- | +| Cue patterns only | 0.777 | 0.518 | 0.439 / 0.0% | 0.701 | 0.521 | 0.39 / 0.47 | +| Cue patterns only +ext | 0.937 | 0.601 | 0.614 / 15.5% | 0.929 | 0.745 | 0.37 / 0.44 | +| Word TF-IDF | 0.526 | 0.377 | 0.249 / 0.0% | 0.714 | 0.463 | 0.22 / 0.25 | +| Word TF-IDF +ext | 0.762 | 0.571 | 0.517 / 0.6% | 0.996 | 0.917 | 0.27 / 0.36 | +| Character n-grams | 0.563 | 0.320 | 0.180 / 0.0% | 0.755 | 0.432 | 0.35 / 0.44 | +| Character n-grams +ext | 0.861 | 0.643 | 0.648 / 8.4% | 0.997 | 0.949 | 0.33 / 0.42 | +| Character n-grams + cues (previous recipe) | 0.735 | 0.525 | 0.450 / 0.0% | 0.802 | 0.495 | 0.76 / 0.85 | +| **Character n-grams + cues +ext (served)** | **0.977** | **0.811** | **0.855 / 7.2%** | 0.996 | **0.972** | 0.72 / 0.86 | +| MiniLM embeddings + LR | 0.494 | 0.286 | 0.165 / 13.4% | 0.821 | 0.502 | 8.9 / 11.5 | +| MiniLM embeddings + LR +ext | 0.626 | 0.459 | 0.398 / 14.3% | 0.969 | 0.746 | 9.7 / 11.9 | -From `backend/artifacts/intel/report.json` (seed 7), which `make intel` writes -and the console page reads. +MiniLM is `paraphrase-multilingual-MiniLM-L12-v2` (sentence-transformers), frozen, +with a logistic regression on its 384-number embedding; it is an optional extra +(`uv sync --extra transformer`) and never a dependency of the service. It lost on +every test set and is 12–14× slower. The likely reason: Banglish is written in +Latin letters the embedding model reads as noise, and the scam signal sits in +short phrases ("code ta bolun") a sentence embedding averages away. -| | New wording of trained scripts (`test`, 1,277 messages) | Scripts never seen (`unseen`, 1,089 messages) | -| --- | --- | --- | -| ROC-AUC | 0.989 | 0.922 | -| Scam messages flagged, classifier alone | 86.8% | 68.2% | -| Scam messages flagged, with the link check | **90.2%** | **81.0%** | -| Harmless messages flagged | 0.23% (hard look-alikes: 0.34%) | 0.0% | +Choice: **character n-grams + cues, trained with the public corpus.** It is best +on all three test sets in macro-F1 and stays under a millisecond. The fallbacks +are the same model trained on our scripts alone (no network at training time) and, +when no classifier is loaded or `cues.yaml` changed since training, the link check +alone. + +**Why it says so.** Each matched cue now carries the exact phrase(s) that matched, +as `{text, start, end}` spans of the customer's own text, in whichever script it +was written: "ekhoni OTP code ta bolun", "ওটিপি কোডটি বলুন", "OTP টা বলুন". A flagged +message also returns up to five `highlights`: the words that pushed the score up +most, read from the character n-gram weights. Both are additive fields; every +earlier field of `message-check` is unchanged. The cue patterns gained two +code-mixed forms (an English secret word followed by a Bangla verb, an English fee +word followed by a Bangla "send"); they were written by the same author as the +rest. + +**The message and the payment.** When a check is flagged, the wallet that asked +is remembered for 30 minutes with only the wallet IDs, phone numbers and amounts +the message contained (at most five messages a wallet), never the text. A payment +that wallet then makes **to one of those numbers, or for one of those amounts +(±1 Tk)**, is raised from `allow` to `warn` with the reason "follows a message +flagged as a likely scam" in English and Bangla; a payment the policy already +stopped keeps its tier and gains the reason. The recipient check says the same +before the amount is typed. The memory lives in the scoring process and is lost +on restart; with several API processes, only the one that served the check knows. -By language, classifier alone (scam flagged / harmless flagged): +## 5. Measured + +From `backend/artifacts/intel/report.json` (seed 7, model `v2`), which +`make intel` writes and the console page reads, and +`backend/artifacts/intel/compare.json` for the comparison above. -| | Banglish | Bangla | English | +| Served model | `test` (1,591) | `unseen` (1,950) | `external` (1,401) | | --- | --- | --- | --- | -| `test` | 81.3% / 0.56% | 91.8% / 0.0% | 92.8% / 0.0% | -| `unseen` | 71.1% / 0.0% | 82.4% / 0.0% | 60.0% / 0.0% | +| ROC-AUC | 0.994 | 0.971 | 0.996 | +| Macro-F1 at `caution` | 0.977 | 0.811 | 0.972 | +| Scam messages flagged (`caution`) | 96.9% | 85.5% | 97.0% | +| Harmless messages flagged (`caution`) | 0.36% | **7.2%** (hard look-alikes 9.3%) | 2.5% | +| Scam messages at `high` | 84.3% | 48.4% | 80.6% | +| Harmless messages at `high` | 0.0% | 0.0% | 0.48% | -Naming the category (a message can have several): +The model this replaces, on the same `external` test set: AUC 0.815, macro-F1 +0.659, 39.6% of scams flagged, 7.6% of harmless messages flagged. -| Category | `test` support | precision | recall | `unseen` support | precision | recall | -| --- | --- | --- | --- | --- | --- | --- | -| 1 Social engineering | 420 | 0.997 | 0.798 | 183 | 0.394 | 0.579 | -| 2 Impersonation | 394 | 0.912 | 0.893 | 81 | 0.421 | 0.951 | -| 3 Fake payment | 50 | 1.000 | 0.820 | 106 | 1.000 | 0.566 | -| 4 Account takeover | 233 | 0.683 | 0.768 | 159 | 1.000 | 0.843 | -| 5 Merchant fraud | 113 | 0.934 | 0.504 | 97 | 0.151 | 0.134 | -| 6 Phishing and malware | 158 | 1.000 | 0.804 | 119 | – | 0.000 | -| 7 Financial network | 77 | 0.825 | 0.857 | 100 | – | 0.000 | -| 8 Scam campaign | 289 | 0.817 | 0.882 | 239 | 0.252 | 0.167 | - -Held-out scripts, share flagged by the classifier alone: customs parcel 99%, bank -card 95%, overpayment 84%, charity visa 83%, eSIM porting 73%, P2P mule 70%, -rental and ticket 53%, utility-bill link **0%**. The three harmless held-out -scripts: 0%. +By language, macro-F1 at `caution`: + +| | Bangla | Banglish | English | Code-mixed | +| --- | --- | --- | --- | --- | +| `unseen` | 0.717 | 0.738 | 0.833 | 0.995 | +| `external` | 0.977 | 0.980 | 0.964 | 0.966 | + +On `unseen`, scam flagged / harmless flagged: Bangla 78.0% / 0.0%, Banglish +86.7% / **28.8%**, English 85.0% / 0.0%, code-mixed 100% / 1.6%. + +By typology on `unseen` (that typology's scams against all harmless messages of +the split), macro-F1 and share of its scams flagged: + +| Typology | macro-F1 | flagged | +| --- | --- | --- | +| Fake agent or helpline | 0.937 | 98.6% | +| Prize or lottery | 0.935 | 97.9% | +| Wrong-number send-back | 0.830 | 71.9% | +| Job or investment | 0.943 | 100% | +| Fake government aid | 0.936 | 99.2% | +| OTP or PIN phishing | 0.894 | 85.4% | +| Other | 0.835 | 78.4% | + +Held-out scripts, share flagged: bank card 100%, job deposit 100%, utility-bill +link 100%, government aid card 99%, helpline search 99%, prize quiz 98%, eSIM +porting 92%, P2P mule 84%, overpayment 83%, customs parcel 79%, rental and ticket +69%, OTP over social media 69%, wrong recharge 64%, charity visa 63%. Harmless: +split bill 0%, school notice 0%, **official notice 21%**. How to read this: -- **Spotting a scam generalises; naming it does not.** On scripts it never saw the - model still flags most scam messages with no false alarms, but puts them in the - wrong category more often than the right one for 5, 6, 7 and 8. When no category - is clear the customer is shown the general advice instead. -- **The utility-bill link script is missed entirely by the classifier** and caught - by the link check. That is why the two run together, and why the combined row - is the one that matters for category 6. -- **Banglish is the weakest language on trained scripts** (spelling varies most), - and English on unseen ones. -- **Merchant fraud is the weakest category**: half named correctly on new wording, - 13% on new scripts. An advance-payment request reads much like an honest seller's. +- **The public corpus is what moved the numbers.** Every candidate gained from + it, on our held-out scripts as well as on the external test. Text written by + other people taught the model spellings ours did not have. +- **The `external` numbers are probably flattering.** The corpus is fairly + uniform in style and about 4% of its test messages also appear in its training + split. Treat 0.97 as an upper bound for text of that kind, not a forecast. +- **The cost is false alarms on scripts it has never seen**: 7.2% of harmless + `unseen` messages, almost all Banglish, mostly the official-notice script (a + real bank or office telling you about a deadline). The model trained on our + scripts alone flagged none of them but caught only 45% of unseen scams. A + `caution` shows advice, it blocks nothing; `high` raised no harmless message on + either of our test sets. +- **Wrong-number send-back is the hardest typology** (72% flagged): the message + is short and polite and asks for nothing secret. The message-to-payment link is + aimed at exactly this case: the amount and number it names are what gets paid. +- **Tuning disclosure.** After the first run on `test`, two kinds of harmless text + were too often flagged (a family member asking for money in Banglish, a + delivery notice with a tracking link), and eight harmless templates of those + kinds were added. `test` is therefore not fully untouched. `unseen` and + `external` were not used for any change. The ledger check has no accuracy figure: it is exact against the ledger by -construction. Its behaviour is covered by tests (`test_intel.py`, -`test_platform.py`): right receiver, wrong amount, not completed, someone else's -payment, a non-numeric ID. +construction. Its behaviour, the cue spans, the highlights and the +message-to-payment link are covered by tests (`test_intel.py`, +`test_platform.py`). ## 6. Limits, stated plainly -- **The corpus is synthetic and its author wrote the cues.** Templates were +- **Our corpus is synthetic and its author wrote the cues.** Templates were written for this project by the same person who wrote the cue patterns, with the trained scripts in view. The held-out scripts were not used to tune them, but - real messages will match less often than these numbers suggest. Before use it - needs real, consented, labelled messages. + real messages will match less often than these numbers suggest. The public + corpus is real text written by other people, but it is one dataset, of + uncertain label quality, with near-duplicates across its splits. Before use it + needs real, consented, labelled messages from this platform's customers. +- **Banglish false alarms on new scripts.** 29% of harmless Banglish messages in + `unseen` reached `caution`. Watch this first in shadow use. +- **The message-to-payment link is in memory.** It is lost on restart and not + shared between API processes; a shared store would be needed to run more than + one. It matches numbers and amounts only, so a scam message that names neither + does not raise the payment. - **A forged cash-in SMS cannot be told from a real one by its text.** Category 3 rests on the ledger check, and that only helps a customer who asks. - **Text only.** No OCR of screenshots, no voice calls, no QR images. A fake QR diff --git a/docs/INGEST.md b/docs/INGEST.md new file mode 100644 index 0000000..4ec5a39 --- /dev/null +++ b/docs/INGEST.md @@ -0,0 +1,203 @@ +# Ingest — the signed webhook for a core-banking system + +`/v1/score` and `/v1/events` need a service account's bearer token. A core-banking +or switch system usually works the other way round: it pushes each payment to a +fraud engine as it is accepted, signs the request with a shared key, retries +until it gets an answer, and wants the verdict back on its own endpoint. That is +this webhook. It takes ISO 20022 pacs.008 credit transfers (in JSON), so a bank +can send what its payment rail already produces. + +Code: `backend/src/fraudlens/platform/ingest.py` (signatures, replay, +idempotency, the pacs.008 model), `api/routes/ingest.py`, `platform/callbacks.py`, +`platform/keys.py`. Partner SDK (Python standard library only): +`sdk/python/fraudlens_partner.py`, with `sdk/python/example.py`. + +## 1. Endpoints + +| Endpoint | Body | Answer | +| --- | --- | --- | +| `POST /v1/ingest/transactions` | pacs.008 with exactly one `CdtTrfTxInf` | `202` queued for the stream | +| `POST /v1/ingest/transactions?wait=decision` | the same | `200` the decision now, the same body as `/v1/score` plus `end_to_end_id` | +| `POST /v1/ingest/transactions/batch` | pacs.008 with 1 to 500 `CdtTrfTxInf` | `202` all queued, in order | + +The queued answer: + +```json +{"msg_id": "MSG-20260415-0001", "accepted": 2, "duplicates": ["E2E-881"], "callback": true} +``` + +`duplicates` lists the `EndToEndId`s of transactions whose `TxId` was already +received in the last 7 days. They are not queued again; the rest of the batch is. + +The endpoints exist only when `FRAUDLENS_INGEST_KEYRING` names a keyring file; +otherwise they answer `404`. + +## 2. Signing a request + +Every request carries four headers: + +| Header | Value | +| --- | --- | +| `X-FraudLens-Key-Id` | the partner's key id, e.g. `upay-20261007-9ef5` | +| `X-FraudLens-Timestamp` | Unix seconds when the request was signed | +| `X-FraudLens-Nonce` | 16 to 64 of `A-Z a-z 0-9 - _`, new for every request | +| `X-FraudLens-Signature` | `v1=` + hex HMAC-SHA256 of the string below | + +What is signed is the timestamp, the nonce, the method, the path with its query, +each followed by a newline, and then the raw body bytes: + +``` +1776234000\n +4f1c0d6a9b2e47c8a1f3\n +POST\n +/v1/ingest/transactions?wait=decision\n +{"GrpHdr":{...},"CdtTrfTxInf":[...]} +``` + +The method and path are signed so that a request made for the queue cannot be +replayed as a request for a decision, or at another endpoint. The body is signed +as sent: sign the exact bytes you put on the wire, not a re-serialisation. + +The server checks, in this order, and stops at the first failure: + +| Check | Refusal | +| --- | --- | +| all four headers present | `401 unsigned` | +| the key id exists, belongs to a partner and is not retired | `401 unknown_key` | +| the timestamp is within ±5 minutes of the server's clock (`FRAUDLENS_INGEST_WINDOW_SECONDS`) | `401 stale_timestamp` | +| the nonce has the right form | `401 bad_nonce` | +| the signature matches (constant-time comparison) | `401 bad_signature` | +| the nonce has not been seen with this key in the last 10 minutes | `409 replayed` | + +A refusal names the check that failed, never the value that was expected. The +nonce is remembered in Redis for twice the window; after that the timestamp check +refuses the same request anyway, so a captured request cannot be sent again. + +## 3. Retries and idempotency + +A retry is a new request: new timestamp, new nonce, new signature. To make it +safe, send `Idempotency-Key` (1 to 128 of `A-Z a-z 0-9 _ . : -`; the SDK uses +the `MsgId`). For 24 hours, per partner: + +- the same key with the same body gets the first answer again, with + `Idempotent-Replayed: true`; refusals such as `409 stale_event` are replayed too; +- the same key with a different body is refused, `422 idempotency_key_reused`; +- while the first request is still being processed, `409 in_progress` with + `retry_after_seconds`; +- a `5xx` or a malformed body is not remembered, so the retry runs. + +Without an idempotency key, the `TxId` check still stops a transaction from being +queued twice; a retried `?wait=decision` for a transaction already decided +answers from the scorer's own duplicate handling. + +## 4. The message: pacs.008 to FraudLens + +JSON with the ISO 20022 element names, the elements FraudLens uses and nothing +else (unknown elements are refused). One `CdtTrfTxInf` is one transaction. + +| FraudLens field | pacs.008 element | Notes | +| --- | --- | --- | +| — | `GrpHdr.MsgId` | 1–35 characters; returned as `msg_id`, the SDK's idempotency key | +| — | `GrpHdr.CreDtTm` | when the message was created | +| — | `GrpHdr.NbOfTxs` | optional; must equal the number of `CdtTrfTxInf` | +| — | `CdtTrfTxInf[].PmtId.EndToEndId` | the bank's reference; echoed in answers and callbacks | +| `txn_id` | `PmtId.TxId` | digits only, below 2^62 | +| `ts` | `AccptncDtTm` | when the payment was accepted, with a time zone | +| `type` | `PmtTpInf.CtgyPurp.Prtry` | `SEND_MONEY`, `CASH_OUT`, `PAYMENT`, … as in `/v1/score` | +| `amount` | `IntrBkSttlmAmt.Amt` | with `IntrBkSttlmAmt.Ccy` = `BDT`; other currencies are refused | +| `sender_id` | `DbtrAcct.Id.Othr.Id` | the wallet or agent number | +| `sender_type` | `DbtrAcct.Id.Othr.SchmeNm.Prtry` | `wallet`, `agent` or `bank` | +| `receiver_id` | `CdtrAcct.Id.Othr.Id` | | +| `receiver_type` | `CdtrAcct.Id.Othr.SchmeNm.Prtry` | `wallet`, `agent`, `merchant`, `telco` or `biller` | +| `sender_balance_before` | `SplmtryData.Envlp.DbtrBalBefore` | | +| `device_id` | `SplmtryData.Envlp.DvcId` | optional | +| `channel` | `SplmtryData.Envlp.Chanl` | `app`, `ussd`, `agent` or `bank` | +| `district` | `SplmtryData.Envlp.Dstrct` | | +| `ip` | `SplmtryData.Envlp.IPAddr` | optional; reduced to its network at once and never stored ([PLATFORM.md](PLATFORM.md) §3) | + +pacs.008 has no place for a wallet's balance, the customer's device or the +channel, so those travel in `SplmtryData`, the standard's extension envelope. +Everything ingested is `source: live`. + +Validation errors are `422 invalid_request`, and each one names the place in the +message, not the FraudLens field, for example +`["CdtTrfTxInf", 0, "IntrBkSttlmAmt", "Ccy"]`. Every transaction in a batch is +checked before any is queued: a batch is accepted whole or not at all. + +A `202` means the transactions are on the stream (`backlog_full`, `503`, when it +is too long). They are decided by the stream worker like `/v1/events`, with the +same ordering and dead-letter rules. + +## 5. Decision callback + +When the partner has a `callback_url` in the keyring, each transaction it queued +is followed by one `POST` to that URL once it has been decided: + +```json +{"txn_id": 8812345678, "end_to_end_id": "E2E-881", "status": "held", "status_reason": null, + "scored": true, + "decision": {"tier": "hold", "action": "hold_for_review", "requires_review": true, + "risk_score": 0.91, "risk_band": "very high", "mode": "...", + "model_version": "...", "policy_version": "...", "decided_at": "..."}} +``` + +It is signed exactly like an ingest request, with the partner's newest key, so +the partner verifies it with the same code (`fraudlens_partner.verify`). The +decision's reasons are not sent: like `/v1/score`, the partner learns what to do, +and the reasons stay with the analysts. + +- **When.** The job waits for the decision, checking every 2 seconds. A + transaction the stream could not process (dead-lettered, stale) is reported + after 10 minutes as `"status": "not_processed"`. +- **Retries.** Any answer other than `2xx`, a timeout (5 s) or a connection error + is retried after 2, 4, 8 … seconds (±20%, at most 10 minutes), 8 attempts in + all. Then the job goes to the Redis list `fraudlens:callbacks:dead` for an + operator. +- **At least once.** The jobs are in a Redis sorted set; whichever API process + removes a due job delivers it, so there is one attempt at a time. A crash + between the `POST` and the bookkeeping can repeat a delivery: treat `txn_id` as + the idempotency key. +- Redirects are not followed. In production the URL must be `https`. + +## 6. Keys + +The keyring is a JSON file, mode `0600`, outside the repository +(`backend/secrets/` is ignored by git): + +```json +{"keys": {"upay-20261007-9ef5": {"secret": "…", "created": "2026-10-07T03:21:34Z", "partner": "upay"}}, + "partners": {"upay": {"callback_url": "https://upay.example/fraudlens/decisions"}}} +``` + +```bash +make partner-key PARTNER=upay CALLBACK=https://upay.example/fraudlens/decisions +# key id: upay-20261007-9ef5 +# secret: 0706… (shown once; hand it over out of band) +make keys # ids, owners and dates, never the secrets +cd backend && uv run python -m fraudlens.platform.keys retire upay-20260901-1a2b +``` + +A new key does not cut the old ones off: they keep verifying for a grace period +(24 hours by default, `--grace-hours`), so the partner can switch over without a +failed request. The newest key signs callbacks. `retire` stops a leaked key at +once. The API re-reads the file when it changes; no restart. + +## 7. Try it + +```bash +make partner-key PARTNER=upay CALLBACK=http://127.0.0.1:9099/fraudlens +FRAUDLENS_INGEST_KEYRING=secrets/ingest.json make api # in another terminal +export FRAUDLENS_PARTNER_KEY_ID=upay-… FRAUDLENS_PARTNER_SECRET=… +python sdk/python/example.py --sender W000123 --receiver W000456 --amount 900 \ + --at 2026-04-15T10:00:00+06:00 # decided now +python sdk/python/example.py ... --queue --listen 9099 # queued; prints the callback +``` + +`--at` sets `AccptncDtTm`. The demo's clock follows the transactions it is sent +([PLATFORM.md](PLATFORM.md) §14), so use a time inside the replayed period. +Outside production a callback URL may be plain `http`. + +Tests: `backend/tests/test_security.py` (signatures, the SDK against the server, +the message mapping) and `backend/tests/test_ingest.py` (bad signature, stale +timestamp, replay, retired key, duplicates, idempotency, batches, callbacks with +retries and the dead list, against Postgres and Redis). diff --git a/docs/MODEL_CARD.md b/docs/MODEL_CARD.md index ede2b93..9fc5605 100644 --- a/docs/MODEL_CARD.md +++ b/docs/MODEL_CARD.md @@ -1,7 +1,7 @@ # Model card — FraudLens risk models, version v4 Every number below is read from `backend/artifacts/models/v4/report.json`, which -`make train` writes, except where §13 says otherwise. Nothing here is +`make train` writes, except where §13 to §17 say otherwise. Nothing here is hand-entered or rounded up. The data is synthetic ([DATA_ASSUMPTIONS.md](DATA_ASSUMPTIONS.md)); absolute values will not transfer to real traffic. @@ -267,8 +267,10 @@ has not been done yet. real fraud. Precision in production would be lower at the same thresholds; thresholds must be re-fitted on real data. - **Labels.** Training uses ground-truth labels. In production only reported - cases are labelled (about half), which biases what the model learns. -- **No adaptation.** Simulated scammers do not react to the controls. + cases are labelled (about half), which biases what the model learns. §15 + measures what that costs and what wins it back. +- **No adaptation.** The scammers behind every number above do not react to the + controls. §14 adds scammers that do, and measures the decay. - **Young wallets and fast cash-out are risk signals.** New customers and remittance receivers legitimately show both. The data contains such customers as hard negatives. This is the main fairness risk, and the fairness report @@ -319,14 +321,63 @@ interrupted (warn or above). Overall it is 0.53%, and 0.04% are held. Groups und receiving wallet is also the strongest honest signal of a mule, so the gap cannot be removed by dropping the feature without losing most of the recall. What limits the harm is the form of the interruption: a warning the customer - can dismiss, and a hold with a 30-minute review deadline. A deployment should - watch this rate and consider a separate, higher threshold for new wallets. + can dismiss, and a hold with a 30-minute review deadline. Policy v3 gives new + wallets cut-offs of their own; what that buys and costs is measured below. - Location, channel and balance gaps are within about a factor of two. No protected attribute exists in the data, so nothing is known about gender, age or religion, and nothing can be claimed. - These are rates on synthetic customers. The report is a method and a place on the dashboard, not evidence about real people. +### Narrowing the gap: policy v3 + +Policy v3 is v2 plus two segments: a transfer to a wallet under 30 days old, and a +payment from one. They are scored against the model's cut-offs times a scale +fitted on the validation period (`make mitigation`; method in +[DECISION_POLICY.md §11](DECISION_POLICY.md#11-young-wallets-segment-thresholds-policy-v3)). +Every rule, hard ones included, still applies, and a held payment still waits +for a person. Measured on the test period (134,545 payments, 25 days) against v2, +on model v4; intervals are 95%, from 1,000 bootstrap resamples of sending +customers. + +| | v2 | v3 | Change (95% interval) | +| --- | --- | --- | --- | +| Senders under 30 days: honest payments interrupted | 3.17% | 0.41% | −3.45 to −2.12 pp | +| … times the overall rate | 5.90 | 1.14 | −5.97 to −3.68 | +| Receiving wallet under 30 days: interrupted | 8.88% | 2.42% | −7.54 to −5.35 pp | +| … times the overall rate (all transfers) | 16.17 | 7.03 | −10.65 to −7.54 | +| Receiving wallet under 30 days: held | 1.50% | 0.70% | −1.17 to −0.47 pp | +| All honest payments interrupted | 0.537% | 0.363% | −0.20 to −0.15 pp | +| Victim transfers alerted | 81.73% | 81.58% | −0.51 to 0.00 pp | +| Victim transfers held | 66.47% | 66.19% | −0.73 to 0.00 pp | +| Victims' money alerted | 85.19% | 84.76% | −1.53 to 0.00 pp | +| All payments held | 0.765% | 0.750% | −0.022 to −0.009 pp | + +- **The trade-off, honestly.** 232 honest payments are no longer interrupted and + 18 honest holds become a step-up instead. In exchange, among fraudulent payments, + one that v2 stepped up is now allowed, one is warned instead of stepped up, and + two that v2 held are stepped up instead (the customer re-authenticates and + waits, but the money is not held for an analyst). Victims' money alerted falls + by ৳15,300 (৳3,024,010 to ৳3,008,710). Precision at warn rises from 64.7% to + 73.1%, and alerts fall from 81.0 to 71.7 a day. +- **It narrows the gap; it does not close it.** Raising the young-receiver warn + cut-off past the guard (×25 the model's, the hold cut-off) would bring the + ratio only from 7.4 to about 6.1 against payments with no young wallet, because + the rest is rules (the mule rule R04 lifts young receiving wallets) and honest + payments the model scores at hold level. Those are not touched: the guard keeps + a payment held for anyone warned in every segment. +- **Intersectional view** (account age × channel × area, in `fairness_mitigation.json` + and on the console). Young receiving wallets on USSD carry the largest remaining + gap: rural ones fall from 29.1× to 16.1× the overall rate (288 honest payments), + urban ones from 30.7× to 22.9× (89 payments, too few to judge). Young senders on + the app in rural areas stay highest among senders (6.9× to 2.7×). Older groups' + ratios rise slightly (for example 0.99 to 1.21) only because the overall rate + they are divided by fell; their own rates all fell. +- The segments were fitted on validation, where they cost no victim recall at all; + the one missed and two softened victim transfers appear only on test. The + default served policy stays v2 until someone chooses v3 + (`FRAUDLENS_POLICY_VERSION=v3`). + ### Verdicts as labels This subsection and the next (shadow mode) were measured with v4 as the served @@ -391,14 +442,381 @@ more wallets sit within three hops of one as time passes. A deployment that keeps this feature should expire or age old flags. The live endpoint measures the same on the latest decisions actually served. -## 14. Reproduce +## 14. Adaptive adversaries + +The scammers in §2 to §13 follow fixed scripts. Real scammers learn which +scripts get through. This section deploys scammers that do, against the served +model (v4), and asks three questions: how fast recall decays, whether weekly +retraining keeps up, and whether the drift monitor notices. + +**Set-up** (`make adversary`, about five minutes; writes +`artifacts/reports/adversary.json`). The default world is re-simulated to day 120 +(it reproduces the stored dataset exactly, which the script checks). Then 8 new +fraud cells open and run 6 one-week rounds after an 8-day warm-up. Every scam +draws one of six tactics: + +| Tactic | What changes | +| --- | --- | +| none | the usual script for the typology | +| split_amounts | each payment split into 2 to 6 parts below ৳1,000 to ৳2,500, 5 to 30 minutes apart | +| seasoned_mule | the money goes to an existing customer's wallet over 180 days old, not a young one | +| delayed_cashout | the mule waits 6 to 48 hours before forwarding or cashing out (past a 30-minute hold) | +| fan_out | each payment goes to the least-used of up to 11 mules | +| mimic_hours | the scam runs at the victim's own usual hour | + +Round 0 runs half the scams on the usual script and spreads the rest evenly. +After each round the cells see which scams had no victim transfer alerted at the +warn tier. They then shift their mix by multiplicative weights on that success +rate (step 3.0, floor 3% per tactic, tactics run fewer than 5 times keep their +weight). Each round is scored in full with no subsampling: 34,000 to 46,000 +transactions and 84 to 117 scams. There are four arms, all on the same world and +seed: + +- **control**: the mix never changes. This is natural decay. +- **frozen**: the scammers adapt and v4 stays deployed. +- **retrain**: after every round the model is refitted with the `make train` + recipe. It uses the training fold plus what an operator would know by then: + the verdict on every alerted transaction, and the complaints of reported scams. + Thresholds are re-chosen on val_b. +- **drift_gated**: retrain only when the live drift monitor + (`mlops/drift.py`: score PSI against training, or the alert rate) leaves "stable". + +**Scams caught (case recall) at the warn tier, by round:** + +| Arm | 0 | 1 | 2 | 3 | 4 | 5 | Pooled PR-AUC | +| --- | --- | --- | --- | --- | --- | --- | --- | +| control | 96.7% | 94.6% | 90.9% | 93.1% | 89.5% | 94.8% | 0.977 | +| frozen | 96.7% | 94.0% | 90.0% | 87.2% | 87.0% | **79.6%** | 0.961 | +| retrain | 96.7% | 97.0% | 90.5% | 92.6% | 97.8% | 98.0% | 0.960 | +| drift_gated | identical to frozen: the monitor never fired | | | | | | 0.961 | + +What the numbers say: + +- **The scammers found the weak spot, and it is seasoned mules.** Against the + frozen model, seasoned_mule's share grew from 10% to 48% of scams. Pooled over + the six rounds, the model caught 74.4% of seasoned-mule scams (191 scams), + against 92% to 100% for every other tactic. The other tactics barely hurt: + splitting, delaying, fan-out and hour-mimicking were each caught at 92% or + more. The model's two largest signals are the receiver's fast-exit share and + its age. Delaying cash-out does not hide the first signal on a mule that has + other victims, and only an old wallet hides the second. +- **Frozen decays, control does not.** By round 5 the frozen model caught 79.6% + of scams against 94.8% for the same model facing non-adaptive scammers. That is + 15 points lost to adaptation alone. Money is more robust: with holds on the + mule's exit transactions, 98.3% of the taka is still covered in round 5. The + scams that get through are caught at cash-out. +- **Weekly retraining keeps recall, at a price in alerts.** The retrained arm + catches 98.0% of scams in round 5 and 90.1% of seasoned-mule scams pooled. But + the first retrain raised alerts from 76 to 234 a day and cut precision from + 72% to 22%. By round 5 it was still at 118 alerts a day and 46% precision, + against 66 and 79% for the frozen model. The feedback rows are mostly alerted + rows and complaints, so they are far richer in fraud than real traffic, and + thresholds re-chosen on val_b do not correct for it. Retraining on verdicts + needs its alert budget re-checked before promotion. That is exactly what + shadow mode (§13) is for. +- **The drift monitor did not notice.** In the frozen arm the population score + PSI stayed between 0.0007 and 0.0044 and the alert rate between 1.29% and + 1.39% (1.24% on validation). So `drift_gated` never retrained and matched the + frozen arm round for round. Fraud is about 1% of traffic, and a tactic shift inside that 1% + barely moves the whole score distribution. Pointed at confirmed fraud only + (val_b fraud as the score reference), the same measure rose from 0.40 in round + 0 to 0.85 in round 5. But the control arm already reads 0.24 to 0.44, so the + standard 0.25 threshold does not separate the two. A fraud-segment monitor + needs its own baseline. The signals that did move are per-tactic catch rates + on confirmed cases, which only labels give. Drift on unlabelled traffic is no + substitute for them. +- **What carries detection after adaptation** (mean |SHAP| share on victim + transfers, round 0, then round 5). The frozen model keeps leaning on the same + features: receiver fast-exit share 14.8% then 12.9%, receiver age 10.8% then + 8.5%. The retrained model moves away from receiver age (to 7.8%) and fast exit + (to 9.5%). It moves toward the sender's handset age (7.7% to 13.2%, now the + largest), the receiver's 7-day fan-in (3.3% to 6.6%), its number of recipients + (3.4% to 6.4%) and prior payments between the pair (3.8%, new in the top 8). + These are the signals an old wallet does not fake: a victim on a new handset, + and many unrelated senders converging on one receiver. + +**Caveats.** One seed and one world. Each round has 84 to 117 scams and each +tactic 4 to 53, so a single round's number moves by a few points on a few scams. +Read the trend, not a single cell. The adversary sees ground truth on what was +alerted, which is more feedback than real scammers get. The tactic set and its +parameters are assumptions (DATA_ASSUMPTIONS.md). The retrained models are +experiment-only and are not registered. + +## 15. Under-reporting: training on reported scams only + +Every number here is read from `backend/artifacts/reports/label_realism.json`, +which `make label-realism` writes (about 5 to 15 minutes). It does not change +the served model or any number above. + +The served model learnt from every scam the simulator ran. A wallet provider +only learns about the ones victims report. To measure what that costs, the +transaction model is retrained, with the same LightGBM settings, training fold +and early stopping as `make train`, on the labels a provider would have had, +and every version is scored on **every** scam in the test period. Each is +compared at the false-positive rate the served model (v4, warn tier) runs at +on test, 0.523% of honest payments, so the rows differ only in how they rank. +Five seeds (7, 11, 13, 17, 19) vary both the reporting draw and the model; +values are mean ± sd. + +**How reporting is simulated.** A case is reported with a probability that +averages the simulator's `report_rate` of 50% but is higher for larger losses, +urban victims, accounts held for a year or more (the data has no person age) +and account takeovers, and lower for lottery-fee and investment scams. Reports +arrive after the simulator's delay (median 1.5 days, 5 for investment scams); +only those in by the start of the test period count. A reported case labels all +its transactions, including the mule's forwards and cash-outs. Unreported fraud +is labelled clean. These weights are assumptions, listed in the JSON, not +measurements. In the training data this labels 82% of takeovers, 56% of +impersonations, 40% of wrong-send and 35% of lottery-fee cases. + +| Training labels | Scams caught | Loss transfers caught | Taka stopped | PR-AUC | +| --- | --- | --- | --- | --- | +| Every scam (as served) | 87.3% ± 0.5 | 81.7% ± 0.7 | 85.0% ± 0.6 | 0.831 ± 0.003 | +| Reported only, half at random | 72.9% ± 5.3 | 66.4% ± 6.0 | 72.3% ± 7.4 | 0.595 ± 0.038 | +| **Reported only, biased** | **78.8% ± 2.7** | 71.4% ± 2.2 | 77.9% ± 1.9 | 0.724 ± 0.019 | +| + positive-unlabelled learning | 79.4% ± 1.5 | 72.1% ± 1.9 | 79.5% ± 1.9 | 0.762 ± 0.020 | +| + same, report rate given as 50% | 79.0% ± 1.8 | 71.9% ± 1.8 | 78.8% ± 2.0 | 0.754 ± 0.020 | +| + labels through mule wallets | 82.8% ± 2.2 | 75.6% ± 1.7 | 80.9% ± 1.5 | 0.770 ± 0.043 | +| + analyst verdicts on alerts | 83.0% ± 2.0 | 76.1% ± 2.1 | 81.9% ± 1.6 | 0.769 ± 0.033 | +| + mule-wallet labels and verdicts | 82.4% ± 1.4 | 75.0% ± 1.4 | 79.7% ± 2.1 | 0.781 ± 0.023 | + +Scams caught, by typology (mean over seeds): + +| Training labels | account_takeover | impersonation | lottery_fee | wrong_send | **investment_scam** (unseen) | +| --- | --- | --- | --- | --- | --- | +| Every scam | 100% | 96.8% | 98.5% | 100% | 71.2% ± 1.3 | +| Reported only, biased | 100% | 96.8% | 95.8% | 100% | **52.4% ± 5.8** | +| + positive-unlabelled | 100% | 96.8% | 95.2% | 100% | 54.1% ± 3.5 | +| + labels through mule wallets | 100% | 96.8% | 96.7% | 100% | 61.5% ± 4.9 | +| + analyst verdicts | 100% | 96.8% | 96.7% | 100% | 61.8% ± 4.5 | + +What this shows: + +- **Under-reporting costs about 8.5 points of scams caught** (87.3% to 78.8%) + at the same false-alert rate, and PR-AUC falls from 0.831 to 0.724. +- **It hides the scam the model has not seen, not the ones it has.** Known + typologies stay at 96 to 100% caught: scripted fraud is easy enough that half + the examples suffice. The held-out investment scam falls from 71.2% to 52.4%. + With fewer examples the model learns the narrow signature of each reported + pattern rather than the general shape of a mule chain, which is what caught + the new scam. +- **Who reports matters less than how many.** Biased reporting did better than + reporting at random (78.8% against 72.9%), with far less spread between + seeds, because it keeps the large, cleanly labelled takeover and + impersonation cases; it also labels more fraud rows (1,293 against 1,066 on + average). The random-reporting result swings by ±5 points with the draw. +- **Positive-unlabelled learning does little.** Elkan–Noto weighting lifts + PR-AUC to 0.762 but scams caught by under one point, within noise, and + telling it the true report rate does not help. Its assumption, that labels go + missing at random among frauds, is what biased reporting breaks. +- **What helps is the network and the analysts.** Spreading soft labels (weight + 0.7) from reported mule wallets to their other incoming and outgoing money + within three days labels about 755 extra rows, 91% of them really fraud, and + wins back 4 points of scams caught and 9 of the unseen typology. Analyst + verdicts on the top 1% of a model's alerts over the last 15 training days + (about 978 reviews, 230 of them fraud no victim had reported) win back the + same. Together they are no better than either alone: they recover largely the + same rows. Roughly half the cost of under-reporting is recovered; the unseen + typology stays about 10 points below the ground-truth model. + +Cautions: the reviewers are simulated and always right (as in §13); the +operating threshold is set on test negatives so every row has the same false- +alert rate, which isolates ranking but is not how a threshold would be chosen in +production; the report propensities are assumptions; and five seeds give +standard deviations of 1.5 to 5 points, so differences under about 3 points +between recovery methods are not meaningful. + +## 16. External validation + +Everything above comes from our own simulator. This section asks two questions: +how precise are the headline numbers, and does the modelling approach hold up on +fraud data we did not generate? `make external` reproduces all of it (it downloads +about 560 MB once and verifies SHA-256 checksums; code in +`backend/src/fraudlens/external/` and `backend/src/fraudlens/models/bootstrap.py`). +Numbers are read from `backend/artifacts/models/v4/bootstrap_ci.json`, +`backend/artifacts/external/paysim.json` and `backend/artifacts/external/baf.json`. + +No real MFS transaction data was available to us. Both external datasets are +themselves synthetic, but generated by other teams from real data: PaySim from a +month of logs of a real African mobile-money service, BAF from a real bank's +account-opening data. Neither contains authorised-push-payment scams, so this is +a test of the method, not of the scam-detection claim itself. + +### Confidence intervals on the headline (synthetic test period) + +1,000 bootstrap replicates, percentile 95% intervals. The model, calibration and +thresholds are fixed; only the test sample is resampled, two ways: whole days (25 +of them, which keeps a day's and a scam's transactions together) and scams (each +of the 360 scams as one unit, each legitimate payment as one unit). + +| Warn tier on v4 | Point | 95% CI, by day | 95% CI, by scam | +|---|---|---|---| +| Scams caught (model warn threshold) | 86.9% | 82.7–91.0% | 83.3–90.6% | +| False-alert rate on legitimate payments (model warn threshold) | 0.52% | 0.47–0.58% | 0.49–0.57% | +| Scams caught (policy v1, warn or above) | 87.5% | 83.6–91.3% | 83.9–90.8% | +| False-alert rate (policy v1, warn or above; §13's 0.53%) | 0.53% | 0.47–0.59% | 0.49–0.57% | +| PR-AUC, any fraud-chain transaction | 0.834 | 0.806–0.858 | 0.807–0.858 | + +The headline "86.9% of scams at 0.53% false alerts" mixes two operating points: +86.9% is the model's warn threshold (§5), 0.53% is the served policy's (§13). At +the policy's own point the figures are 87.5% and 0.53%. Either way, with 360 scams +the recall is known to about ±4 points. + +### PaySim (mobile money) + +6,362,620 transactions over 743 hours. Fraud is account takeover: the balance is +TRANSFERred out and cashed out. Only TRANSFER and CASH_OUT rows are scored (the +only types that are ever fraud). Split by hour: train 1–300 (1,780,544 rows, +3,401 frauds), validation 301–400 (748,992; 1,076), test 401–743 (240,873; +3,736). The test hours have far less legitimate traffic, so the base rate there is +1.55% against 0.19% in training. + +Features mapped from FraudLens, each computed from earlier rows only: +receiver-centric (earlier incoming payments, in the last 24 hours and in value; +hours since the receiver last received and since it was first seen; earlier +incoming TRANSFERs; amount against the receiver's usual incoming amount), velocity +(the 24-hour counts; the sender's earlier payments), and cash-out timing (hours +since the sender last received money, amount against what it received). The last +one cannot see PaySim's fraud chain: the account that receives a fraudulent +TRANSFER never appears as the one that cashes out (0 of 4,097). Devices, agents, +districts, wallet age and confirmed-fraud flags have no equivalent. + +Thresholds are fixed on the validation hours at each target false-positive rate +and applied unchanged to test, as in §5. Because the test hours look different, +the rate reached on test drifts from the target; both are shown. The last column +sets the threshold on the test set itself, which no deployment can do but which +makes scores comparable at exactly 0.5%. + +| Score | PR-AUC | ROC-AUC | Recall at 0.5% target (FPR reached on test) | Recall at 1% (FPR reached) | Recall at exactly 0.5% FPR on test | +|---|---|---|---|---|---| +| PaySim's own rule (TRANSFER over 200,000) | 0.019 | 0.502 | 0.3% (0.00%) | 0.3% (0.00%) | 0.3% | +| Amount only | 0.145 | 0.704 | 0.0% (0.01%) | 8.8% (0.13%) | 13.3% | +| Isolation Forest (no labels) | 0.022 | 0.530 | 4.1% (0.90%) | 6.6% (2.02%) | 3.1% | +| Logistic regression, same features | 0.323 | 0.790 | 25.1% (0.26%) | 32.4% (0.54%) | 31.5% | +| **LightGBM, same recipe as v4** | **0.468** | **0.940** | 57.3% (1.25%) | 68.0% (2.03%) | 32.1% | +| LightGBM without hour of day | 0.469 | 0.903 | 32.9% (0.32%) | 46.8% (0.91%) | 36.7% | +| LightGBM plus pre-transaction balances | 0.991 | 1.000 | 99.97% (0.54%) | 99.97% (1.15%) | 99.95% | + +Bootstrap by test hour (343 hours, 500 replicates): LightGBM PR-AUC 0.394–0.555, +recall at the 0.5% validation threshold 53.4–60.7%; without hour of day, PR-AUC +0.420–0.519 and recall 30.9–35.1%. + +What this shows: + +- **The approach beats every simple baseline on external MFS data**: PR-AUC 0.47 + against 0.32 for logistic regression on the same features and 0.15 for amount + alone. The receiver-centric features carry the model: without hour of day the + top features are amount (37% of gain), amount against the receiver's usual + incoming amount (20%), hours since the receiver last received (15%) and the + receiver's earlier incoming payments (12%). +- **At a 0.5% false-positive rate it catches about a third of PaySim fraud** + (32–37%), not 87%. PaySim's fraud is single-step account takeover with no mule + chain, no shared devices and no history to learn a victim's pattern from, which + is where most of the FraudLens signal lives (§10). The number to compare with + our 86.9% is not this one: the tasks differ. +- **Hour of day helps ranking but hurts stability.** PaySim spreads fraud evenly + over the day while legitimate traffic follows a daily cycle, so the model leans + on hour (32% of gain); in the quieter test hours its validation threshold lets + through 1.25% false positives instead of 0.5%. Without it the threshold holds + (0.32%). The same lesson as §11: a signal that is an artefact of the generator + looks strong and does not travel. +- **PaySim's balance fields are a known artefact.** 97.8% of frauds move exactly + the sender's whole balance and no legitimate TRANSFER or CASH_OUT does, so adding + the pre-transaction balances gives near-perfect scores. We report it to show + why published PaySim results near 0.99 say little, and we do not count it. +- Isolation Forest alone is near chance here (ROC-AUC 0.53), consistent with §4, + where the anomaly model is a weak detector on its own. + +### Bank Account Fraud (BAF, NeurIPS 2022), Base variant + +1,000,000 account applications over 8 months, 1.1% fraudulent, from OpenML +(dataset 46793). This is account opening, not payments; the nearest FraudLens task +is the mule-wallet model (wallets opened to receive scam money). Mapped ideas: +application velocity (6 h, 24 h, 4 weeks), postcode and branch velocity, shared +devices and identities (distinct e-mails per device and per date of birth). +Split by month as in the benchmark: train 0–4 (675,666; 6,740 frauds), validation +5 (119,323; 1,411), test 6–7 (205,011; 2,878). All columns are used as given. + +| Score | PR-AUC | ROC-AUC | Recall at 5% target (FPR reached) | Recall at exactly 5% FPR on test | Recall at exactly 1% | +|---|---|---|---|---|---| +| Velocity counts only | 0.014 | 0.486 | 2.8% (3.04%) | 4.9% | 0.9% | +| Credit-risk score only | 0.037 | 0.672 | 19.8% (4.93%) | 19.8% | 6.6% | +| Isolation Forest (no labels) | 0.018 | 0.576 | 8.1% (5.49%) | 7.3% | 1.5% | +| Logistic regression | 0.143 | 0.863 | 46.9% (5.22%) | 45.9% | 20.3% | +| **LightGBM, same recipe as v4** | **0.193** | **0.893** | 58.8% (6.03%) | 54.8% | 25.3% | + +Bootstrap by application (500 replicates): LightGBM PR-AUC 0.179–0.207, recall at +the 5% validation threshold 57.2–60.7%. + +The same recipe, unchanged, beats logistic regression and every single-signal +baseline on a public benchmark built from real bank data. BAF's velocity columns +count applications across the whole population, not per customer, and carry no +signal on their own (ROC-AUC 0.49); the model's top features are housing status, +device OS, name–e-mail similarity and address history, not the FraudLens ideas. +So BAF supports the modelling recipe, not our particular features. + +### What this does and does not establish + +- It establishes that the modelling recipe (gradient-boosted trees on + receiver-centric and velocity features, thresholds fixed in advance on later + validation data) is not an artefact of our simulator: on two independent public + datasets it has the highest PR-AUC of the scores tried (leaving aside PaySim's + balance artefact). At a fixed low false-positive rate its lead over logistic + regression is small on PaySim (32–37% against 31.5% at exactly 0.5%) and clear + on BAF (54.8% against 45.9% at exactly 5%). Intervals were computed for the + LightGBM scores only. +- It does not validate the 86.9% figure. No public dataset contains MFS + authorised-push-payment scams with mule chains, so the scam-specific numbers in + §3–§7 still rest on synthetic data. The test that would settle it is a + back-test on a provider's own labelled transactions. +- Both external datasets are synthetic derivatives of real data, with artefacts of + their own (shown above for PaySim). + +## 17. The scam-message classifier (intel `v2`) + +A separate model, for text a customer pastes in before paying. Numbers in this +section come from `backend/artifacts/intel/report.json` (`make intel`) and +`compare.json` (`make intel-compare`); the full tables, data provenance and limits +are in [FRAUD_TAXONOMY.md](FRAUD_TAXONOMY.md) §4–§6. + +| | | +| --- | --- | +| Model | logistic regression, one per output, over character n-grams (2–5, TF-IDF) and 27 cue patterns with their pairwise products | +| Outputs | scam or not, and the eight fraud categories | +| Training data | 413 synthetic templates in Bangla, Banglish, English and code-mixed text (5,114 training messages), plus the training split of a public MIT-licensed Bangla SMS smishing corpus (4,903 messages, scam-or-not only) | +| Thresholds | per source on harmless validation messages, stricter kept: `caution` ≤ 3% flagged, `high` ≤ 0.5% | +| Cost | about nine seconds to train; 0.72 ms median, 0.86 ms p95 per message on a laptop CPU | +| Chosen over | cue patterns alone, word TF-IDF, character n-grams alone, multilingual MiniLM embeddings + LR (each with and without the public corpus); best macro-F1 on all three test sets | + +| At `caution` | new wording (`test`) | held-out scripts (`unseen`) | public corpus test (`external`) | +| --- | --- | --- | --- | +| ROC-AUC | 0.994 | 0.971 | 0.996 | +| Macro-F1 | 0.977 | 0.811 | 0.972 | +| Scams flagged | 96.9% | 85.5% | 97.0% | +| Harmless flagged | 0.36% | 7.2% | 2.5% | + +Macro-F1 on `unseen` by language: Bangla 0.717, Banglish 0.738, English 0.833, +code-mixed 0.995. The previous classifier scored macro-F1 0.659 on `external`. +The weak point is harmless Banglish on scripts never seen (29% flagged at +`caution`, none at `high`). + +How it reaches the payment: a payment made within 30 minutes of a flagged check, by +the same wallet, to a number or for an amount the message named, is raised from +`allow` to `warn` with the reason shown (`decided_by: message_link`). Only the +numbers and amounts are held, in memory, never the text. +## 18. Reproduce ``` make data features train # about one minute; writes backend/artifacts/models// make policy insights # policy_report.json and insights.json (fairness, drift, threshold sweep) +make mitigation # §13: young-wallet thresholds (policy v3) against v2 +make adversary # §14: adaptive scammers vs frozen, retrained and drift-gated models +make label-realism # §15: reported-only labels and recovery methods +make external # §16: PaySim, BAF and bootstrap intervals (downloads ~560 MB once) make test # 203 tests, including leakage and round-trip checks ``` `report.json` holds every number above; `manifest.json` holds thresholds, calibration, feature list and data seed; `test_scores.parquet` holds the score of every test transaction. + diff --git a/docs/PLATFORM.md b/docs/PLATFORM.md index 78fc2bf..82e05f5 100644 --- a/docs/PLATFORM.md +++ b/docs/PLATFORM.md @@ -70,6 +70,7 @@ served at `/docs` outside production. | Auth | `POST /auth/login`, `GET /auth/me`, `POST /auth/logout` | anyone / signed in / signed in | | Demo sign-in (§7, off by default) | `GET /auth/demo`, `POST /auth/demo-login` | anyone, when enabled | | Scoring | `POST /score`, `POST /events` (up to 500 per call, queued), `POST /wallet-flags` | service | +| Ingest ([INGEST.md](INGEST.md)), off without a partner keyring | `POST /ingest/transactions[?wait=decision]`, `POST /ingest/transactions/batch` (pacs.008, HMAC-signed, no token) | partner key | | | `POST /score/what-if` | service, analyst, supervisor | | Alerts | `GET /alerts`, `GET /decisions/{txn_id}`, `GET /decisions/{txn_id}/narrative?lang=en\|bn`, `GET /stream/alerts` (server-sent events) | analyst, supervisor | | Cases | `GET /cases`, `POST /cases`, `GET /cases/{id}`, `POST /cases/{id}/assign\|notes\|escalate\|verdict`, `GET /users/reviewers` | analyst, supervisor | @@ -79,12 +80,14 @@ served at `/docs` outside production. | Network | `GET /wallets/{id}`, `GET /wallets/{id}/network`, `GET /rings`, `GET /rings/{ring_id}`, `GET /agents/risk`, `GET /agents/{id}`, `GET /past-cases/{id}` | analyst, supervisor | | Customer | `POST /customer/recipient-check`, `POST /customer/transactions/{txn_id}/respond`, `POST /customer/reports` | service | | | `POST /customer/message-check`, `POST /customer/payment-verify` ([FRAUD_TAXONOMY.md](FRAUD_TAXONOMY.md)) | service | +| | `POST\|GET /customer/transactions/{txn_id}/appeal` (§6) | service | +| Appeals | `GET /appeals`, `POST /appeals/{id}/approve\|reject` (§6) | analyst, supervisor | | Fraud types | `GET /intel/taxonomy` | analyst, supervisor, admin | | Operations | `GET /metrics/summary`, `GET /metrics/daily`, `GET /model`, `GET /model/report`, `GET /policy` | analyst, supervisor, admin | | After deployment (§12) | `GET /models`, `GET /model/shadow`, `GET /metrics/drift`, `GET /feedback` | analyst, supervisor, admin | | Audit | `GET /audit` | supervisor, admin | | | `POST /audit/reveals` (§7) | analyst, supervisor | -| Demo (§14), absent in production | `GET /demo/scenarios`, `GET /demo/payment-draft`, `GET /demo/payment-claims`, `POST /demo/pay\|respond\|report\|recipient-check\|message-check\|payment-verify\|advance-clock` | analyst, supervisor, admin | +| Demo (§14), absent in production | `GET /demo/scenarios`, `GET /demo/payment-draft`, `GET /demo/payment-claims`, `GET /demo/appeal`, `POST /demo/pay\|respond\|report\|appeal\|recipient-check\|message-check\|payment-verify\|advance-clock` | analyst, supervisor, admin | | | `GET /health`, `GET /ready` | none | Errors always have the same shape, with the request id that is also in the @@ -109,6 +112,11 @@ repeat the value that was sent. `POST /v1/events`. They keep the feature state current. 5. When your own investigation confirms a wallet as fraud: `POST /v1/wallet-flags`. +A core-banking system that would rather push than call with a token uses the +signed webhook instead of steps 2 and 4: ISO 20022 pacs.008 credit transfers, +HMAC-signed per request, with replay protection, idempotency keys and an optional +signed decision callback ([INGEST.md](INGEST.md)). + A transaction may carry `ip`, the address the customer's request came from (optional, IPv4 or IPv6). The platform reduces it to its network at the edge (the /24 of an IPv4 address, the /48 of an IPv6 one) and keeps only that, in @@ -172,6 +180,25 @@ and each request still needs its own approval by a second person. from the SMS or screenshot shown, and only for payments made to the asking wallet. 10 a minute per wallet. Both are described in [FRAUD_TAXONOMY.md](FRAUD_TAXONOMY.md). +- **Which scam it looks like.** `/v1/score` and the demo payment return + `decision.scenario` and `decision.cue` (`reported_recipient`, `impersonation`, + `prize`, `investment`, `wrong_send`, `not_you` or `generic`), so the app can + word its warning for that scam (`decision/customer.py`). The phone demo does, + in Bangla and English, with a live cooling-off or review countdown and a "what + happens next" timeline. +- **Appealing.** `POST /customer/transactions/{txn_id}/appeal` (the sender only, + on a warned, step-up or held payment; one per payment; 5 an hour) with a + relation to the recipient and a reason (digits redacted). A person answers + every appeal: `GET /appeals`, `POST /appeals/{id}/approve|reject` (analyst, + supervisor; same escalation rules as the case). Deadlines: the hold's review + SLA (30 minutes) for a held payment, 24 hours for a warning. Approving a held + payment releases it, except into a confirmed-fraud or frozen wallet (`409`). + Approving a warning moves no money and grants no cooling-off waiver, so an + appeal cannot be used to coach a victim past the check. An approved appeal is a + `legitimate` label in `/feedback` (`source: appeal`) unless a case verdict says + otherwise; a rejected one adds nothing. The customer's view + (`GET /customer/transactions/{txn_id}/appeal`) shows status and deadline only. + Everything is audited. Migration `0004_appeals`. ## 7. Security @@ -214,7 +241,9 @@ and each request still needs its own approval by a second person. - **Production guard.** With `FRAUDLENS_ENVIRONMENT=production` the API refuses to start with the development signing secret, the development database password, a Redis URL without a password, `*` as a CORS origin, or demo sign-in enabled, - and the interactive docs are off. + and the interactive docs are off. It also refuses keyring keys shorter than 32 + characters, partner callbacks that are not `https`, and a trusted-proxy list + that trusts everyone (`0.0.0.0/0`). - **Demo sign-in.** Off by default. `FRAUDLENS_DEMO_LOGIN=true` makes `POST /auth/demo-login` sign in as one of the five seeded console accounts without a password, and the login page shows a button for each. Anyone who can @@ -222,6 +251,22 @@ and each request still needs its own approval by a second person. you can reach. Service accounts are never offered, each use is audited as `auth.login` with `demo: true`, and the endpoint answers 404 when the setting is off or the environment is production. +- **Key rotation.** Each token names its signing key (`kid`). With + `FRAUDLENS_JWT_KEYRING` set, `make rotate-jwt` adds a key that signs from then + on, and the earlier ones keep verifying until the tokens they signed have + expired, so nobody is signed out. Partner HMAC keys rotate the same way + (`make partner-key`, with a grace period), and `keys retire` cuts off a leaked + key at once. The keyring files are re-read when they change; no restart. +- **Secrets from files.** Every setting can come from a file: + `FRAUDLENS_JWT_SECRET_FILE=/run/secrets/jwt` (Docker or Kubernetes secrets), + never both the value and the file. `FRAUDLENS_SECRETS_PROVIDER=module:function` + plugs in a secrets manager; only a directory reader ships + (`fraudlens.secret_sources.directory_provider`). +- **TLS and the client address.** `make tls` puts Caddy in front of the API and + the console (HTTPS, HSTS, a 2 MB body cap; `deploy/Caddyfile`). The API believes + `X-Forwarded-For` only from the addresses in `FRAUDLENS_TRUSTED_PROXIES`, and + only as far back as those proxies wrote it; a client's own header is ignored. + The threat model is [SECURITY.md](SECURITY.md). - **Redis password.** Optional in development: `REDIS_PASSWORD` in a `.env` next to `docker-compose.yml` turns it on, and `FRAUDLENS_REDIS_URL` carries the same password. Without it Redis relies on its port being bound to 127.0.0.1. @@ -230,9 +275,12 @@ and each request still needs its own approval by a second person. ## 8. Stream and live feed -`POST /v1/events` appends to a Redis stream; a worker thread in the API process -reads it through a consumer group, in order, in batches, and acknowledges after -the batch is committed. An entry that cannot be parsed or fails validation goes +`POST /v1/events` appends to a Redis stream. Workers read it through one consumer +group: a thread in the API process, and as many `fraudlens.platform.worker` +processes as the load needs (`make worker`). They put entries into the feature +state in stream order and decide them in parallel; an entry is acknowledged and +deleted once its decision is committed, and one held by a worker that died is +taken over by another. SCALING.md has the design and the measurements. An entry that cannot be parsed or fails validation goes to a dead-letter stream with the reason and does not block the ones behind it. Entries delivered before a crash are handled first after a restart, and since scoring is idempotent a redelivery is harmless. If the backlog passes 200,000 the @@ -256,15 +304,18 @@ written before they existed is refused at load (`make features` rebuilds it). ## 10. Limits, stated plainly -- **One scorer process.** The feature state is in one process's memory, so the - API cannot be scaled by adding copies. Scaling out means partitioning by wallet - or moving the state to a shared store; neither is built. +- **One ordered step.** Every process holds the whole feature state and catches + up from the log, so workers and API copies can be added, but putting events + into the state stays serialised behind one lock. Measured in SCALING.md: + throughput stops growing at a few workers. Going further means partitioning + the state by wallet, which is not built. - **Recovery time grows** with the number of transactions since the snapshot. A production system would write snapshots periodically; this one has only the end-of-history snapshot. -- **Rate limits and the login lockout use the address the API sees.** Behind a - proxy, run uvicorn with `--proxy-headers` and a trusted proxy list, or every - client shares one address. +- **Rate limits and the login lockout use the client address** worked out from + `FRAUDLENS_TRUSTED_PROXIES`. Left empty behind a proxy, every client shares the + proxy's address; listing a proxy that passes on its clients' own headers lets + them choose their address. - **The live feed authenticates with the Bearer header**, which the browser's `EventSource` cannot send; the console reads it with `fetch`. - **Step-up is asserted by the caller** (`step_up_passed`). The platform trusts @@ -277,8 +328,16 @@ written before they existed is refused at load (`make features` rebuilds it). sent to the language model is masked in the backend, before it leaves. - **The free-text mask is a pattern match.** It catches long numbers and e-mail addresses, not a name or an address written in words. -- **No TLS, secrets manager or key rotation**: deployment concerns outside this - prototype. +- **Security is partly built** ([SECURITY.md](SECURITY.md) lists each threat as + done, partial or not done). TLS ends at the proxy: the hops behind it (API, + Postgres, Redis) are plain on a private network. Secrets come from files or a + provider hook, but no secrets-manager client ships. Keys rotate when someone + runs the command; nothing rotates them on a schedule, and moving from the single + `FRAUDLENS_JWT_SECRET` to a keyring signs everyone out once. +- **Decision callbacks** cover transactions queued through the ingest webhook, + not `/v1/events`. Delivery is at least once, and the outbound request does not + check where the partner's address resolves to: callback URLs are set by an + operator, not by the partner. ## 11. Measured @@ -331,7 +390,7 @@ same two rows: the figures depend heavily on what else the machine is doing. 292,567 events in just under six minutes, about 855 events a second. In the earlier run eight concurrent what-if callers got 194 answers a second with a p95 of 51 ms: scoring is serialised in one process, so concurrency adds waiting, not -capacity (§10). For scale: the simulated system averages 0.14 transactions a +capacity (§10). Several stream workers are measured in SCALING.md. For scale: the simulated system averages 0.14 transactions a second. **Restart.** With all 304,867 events since the snapshot to re-apply, the service diff --git a/docs/SCALING.md b/docs/SCALING.md new file mode 100644 index 0000000..dd0128b --- /dev/null +++ b/docs/SCALING.md @@ -0,0 +1,145 @@ +# Scaling the scorer + +How the event stream is consumed by several worker processes at once, what that +gains, and where it stops. The code is in `backend/src/fraudlens/platform/` +(`stream.py`, `worker.py`, `scoring.py`); the deployment files are in `deploy/` +and `docker-compose.scale.yml`. + +## 1. The constraint + +A transaction's features depend on every transaction and flag before it: the +sender's velocity, the recipient's fan-in, the agent's cash-out pattern. The +state that holds them lives in memory, and the database is its log (PLATFORM §9). +Two workers that each applied half the stream would each hold the wrong state. +So the design keeps one order for changing the state, and moves everything else +out of it. + +## 2. Every process is a replica + +The API and every worker load the model and the policy once, at start, and +rebuild the whole feature state from the snapshot plus the log. Before a process +changes the state it takes a Postgres advisory lock and replays whatever other +processes have applied since it last looked (rows with a higher `applied_seq`), +so it always starts from the latest state. An approved freeze or release made +through the API reaches the workers the same way, and the API follows the log +in the background (`FRAUDLENS_FOLLOW_INTERVAL_S`), so its `/v1/score` answers +see what the workers did. + +## 3. Two steps per batch + +Workers share one Redis consumer group, `scorer`, on `fraudlens:events`. + +1. **Ordered: into the state.** A worker may put a batch into the state only once + the entry just before it has been put in by someone. The Redis key + `:sequenced` holds the last entry that has; a Lua script moves it + forward and never back. Under the advisory lock the worker catches up, drops + duplicates, computes features, applies the events and writes the transaction + rows, plus a `pending_decisions` row holding each transaction's features. It + then commits and releases the lock. This step is the short one. +2. **Parallel: the decision.** Outside the lock the worker runs the models, the + policy and the explanations, which is most of the cost. It then claims its + rows by deleting them from `pending_decisions` (`DELETE … RETURNING`) and, + in the same transaction, writes decisions, cases and shadow scores. Cases are + per subject, so it takes a lock per subject, in sorted order. After the + commit it publishes alerts, then acknowledges and deletes the stream entries. + +## 4. Exactly once + +Delivery is at least once and the effects happen once: + +- **Duplicates never reach the state.** A transaction id already in the + database is a duplicate, and is checked under the lock. +- **A decision is written only by whoever deletes the pending row.** If two + workers end up with the same entry, after a reclaim racing a slow worker, the + loser deletes nothing and writes nothing. The test + `test_two_workers_holding_one_entry_record_one_decision` drives this race. +- **Crashing between the steps loses nothing.** The pending row survives with + the features exactly as computed, so whoever takes the entry over decides from + them without touching the state again. + +## 5. A worker that dies + +Every worker runs `XAUTOCLAIM` every `claim_idle_ms / 2` (30 s by default), +including while it waits for an earlier entry to be sequenced. Entries idle in +a dead consumer's pending list move to a live one and are handled one by one, in +order. A pod name is a consumer name, so a restarted pod is a new consumer and its +predecessor's entries are claimed like any other's. The test +`test_a_stopped_workers_entries_are_taken_over_in_order` stops a worker holding +entries and checks that the others finish them, in order. The stream-replay +tests drain the test period with two workers and require the served features +and tiers to match the offline evaluation exactly. + +## 6. Measured + +`make loadtest` (`fraudlens.platform.loadtest`) resets the database to the +historical period, starts N worker processes, and: + +- **Saturation.** Writes the first 20,000 test-period events at once and times + how long it takes until the last one is decided. +- **Open loop.** Writes the next 9,000 at a fixed 200 a second. It records each + entry's end-to-end latency, from the entry being added to its decision being + committed. +- **Parity.** Runs `verify`, comparing every served decision with the offline + evaluation. + +Machine: Apple M5, 10 cores, 16 GB, macOS 27.0, Postgres 16 and Redis 7 in +Docker. Other jobs were running on it: the load average was 21 to 28 throughout, +so absolute numbers are pessimistic and noisy. Raw results: +[`scaling/loadtest-20261007-094657.json`](scaling/loadtest-20261007-094657.json). + +| Workers | Throughput (events/s) | Speed-up | p50 (ms) | p95 (ms) | p99 (ms) | Tier mismatches | Differing feature rows | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 1 | 334 | 1.00x | 54.1 | 439.3 | 1014.4 | 0 | 0 | +| 2 | 579 | 1.73x | 40.5 | 194.1 | 291.7 | 0 | 0 | +| 4 | 931 | 2.78x | 105.6 | 192.2 | 287.7 | 0 | 0 | + +Each run compared 14,419 decisions, and `verify` passed every time. The work +split evenly: 14,540 and 14,460 entries with two workers, and 6,951 to 7,530 +each with four. + +What the numbers say: + +- **Throughput grows with workers, sub-linearly.** The parallel step is most + of the work, so two workers nearly double it. At four, the ordered step and + the per-subject case locks start to show. +- **Tail latency falls sharply.** One worker at 200 a second runs at about 60% + of its capacity, so bursts queue: p99 is one second. With two or more workers + the backlog clears and p99 drops to about 0.3 s. +- **The p50 rises at four workers.** Each batch now waits its turn for the + ordered step, on a machine with more runnable jobs than cores. +- **For scale:** the simulated provider averages 0.14 transactions a second, and + a large real one peaks at a few thousand. + +## 7. Deployment + +- **Compose.** `docker compose -f docker-compose.yml -f docker-compose.scale.yml + --profile demo up -d --scale worker=4` runs the demo with four worker + containers. The API keeps scoring requests and follows the log. +- **Kubernetes.** `deploy/k8s` (`kubectl apply -k deploy/k8s`) contains: + - Deployments for the API and the workers, with startup, liveness and + readiness probes (the worker serves `/health`, `/ready` and `/metrics` on + 8081); + - resource requests and limits; + - a PodDisruptionBudget for each; + - a KEDA `ScaledObject` that scales the workers on the stream's length, from 1 + to 4. + + Workers delete entries once decided, so the length is exactly the work still + to do. `hpa-external.yaml` is the same policy as a plain HPA on an external + metric, for clusters without KEDA. The manifests pass `kubeconform -strict` + against Kubernetes 1.30 and the KEDA CRD schemas. + +## 8. Limits, stated plainly + +- **The ordered step is a ceiling.** Events enter the state one batch at a time, + so throughput stops growing at a few workers. This is why the scaler stops at + 4. Going further means partitioning the state by wallet, so that unrelated + wallets sequence independently. That is not built, and it is hard because a + transaction touches two wallets and an agent. +- **Every process holds the whole state.** Memory grows with the number of + replicas, and so does start-up replay time. +- **Measured on one busy laptop,** with Postgres and Redis on the same machine. + The manifests have not been run on a real cluster. +- **The live feed is at most once.** A worker that crashes after its commit but before + publishing loses that alert on the live feed. The decision and the case are in + the database, and the console shows them. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..8ed4183 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,162 @@ +# Security — threat model, personal data and retention + +What FraudLens protects, from whom, and how far each protection goes. Each +threat is marked **done** (built and tested), **partial** (built, with a gap +named) or **not done**. The controls themselves are described in +[PLATFORM.md](PLATFORM.md) §7 and [INGEST.md](INGEST.md). + +## 1. Assets + +| Asset | Where | Why it matters | +| --- | --- | --- | +| Transactions and decisions | Postgres `transactions`, `decisions` | Customers' payments; a changed decision moves or holds money | +| Wallet state (frozen, flagged) | Postgres `wallets`, `freeze_requests`, `wallet_flags` | A freeze stops a customer's money; an unfreeze releases a mule's | +| Cases, notes and verdicts | Postgres `cases`, `case_events` | Evidence, and the labels the next model is trained on | +| Customer scam reports | Postgres `customer_reports` | Free text a customer wrote, possibly with names and numbers | +| Audit log | Postgres `audit_log` (append-only) | The record of who saw and did what | +| Token signing keys | `FRAUDLENS_JWT_SECRET` or the JWT keyring | Whoever has one can be any user | +| Partner HMAC keys | the ingest keyring | Whoever has one can inject transactions as that bank | +| Database and Redis credentials | environment or secret files | Direct access to everything above | +| Models and the decision policy | `backend/artifacts/`, `decision/policies/` | Changing them changes every decision | +| Feature state | the scorer's memory, snapshots in Postgres | Wrong state, wrong scores | + +## 2. Trust boundaries + +``` + customer ──► payment app/upay ──(bearer token: service)──────────┐ + │ + core banking ──(HMAC-signed pacs.008, §INGEST)───────────────────┤ + ▼ + analyst's browser ──(HTTPS, Caddy)──► console ──(bearer token)──► API ──► Postgres + │ └───► Redis (stream, revocations, nonces) + │ + partner callback URL ◄──(HMAC-signed POST) + language model (optional) ◄──(masked evidence) +``` + +1. **Internet to the edge.** Browsers and partners reach Caddy over TLS + (`make tls`). Caddy replaces any `X-Forwarded-For` a client sends. +2. **Edge to the API.** A private Docker network; the API believes forwarded + addresses only from `FRAUDLENS_TRUSTED_PROXIES`. +3. **Caller to the API.** Every request is authenticated: a bearer token with a + role, or a partner's HMAC signature on the ingest endpoints. Nothing else. +4. **API to its stores.** Postgres and Redis on the private network, with + passwords; their ports are bound to 127.0.0.1 on the host. +5. **API outwards.** Decision callbacks to partner URLs set by an operator, and + (off by default) case-note wording from a language model. + +## 3. Threats (STRIDE) + +### Spoofing + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Guessing an analyst's password | Argon2id; five failures in five minutes lock the username from that address; a per-address limit | done | +| A forged or altered token | HS256 with a key of at least 32 characters (production guard); `kid` must name a known, unretired key | done | +| A stolen token | 8-hour expiry, sign-out revokes it in Redis (fails closed), the user is re-read on every request | partial: no binding to a device, no refresh-token rotation | +| A forged ingest request | HMAC-SHA256 over timestamp, nonce, method, path and body; constant-time comparison | done (`test_ingest.py`) | +| Someone posing as FraudLens to a partner's callback endpoint | Callbacks are signed with the partner's own key, verified by the SDK | done | +| Spoofing the client address to dodge the lockout | `X-Forwarded-For` is read only from configured proxies, right to left; `0.0.0.0/0` is refused | done (`test_security.py`) | +| The demo sign-in left on | Off by default, refused in production, every use audited | done | + +### Tampering + +| Threat | Mitigation | Status | +| --- | --- | --- | +| A request changed in transit | TLS at the edge; ingest bodies are signed whatever the transport | partial: TLS ends at the proxy, the private hops are plain | +| Rewriting the audit trail | A trigger refuses `UPDATE`, `DELETE`, `TRUNCATE` on `audit_log` | partial: a database superuser can drop the trigger; no external copy | +| Changing a decision or a freeze | Roles; freezes need a second person (§5 of PLATFORM.md); every step audited | done | +| Injection through input | Every body validated, unknown fields refused, identifiers match a pattern, SQL parameterised, 1 MB body cap (2 MB at the proxy) | done | +| Poisoning the models through analyst verdicts | Retraining registers a challenger only; promotion is a separate, manual step with a shadow comparison | partial: no automatic check for a run of suspicious verdicts | +| Swapping the model or policy files | Loaded once at start-up from the artifacts directory and the image | not done: no signature on artifacts | + +### Repudiation + +| Threat | Mitigation | Status | +| --- | --- | --- | +| "I never froze that wallet" | Every case action, freeze step, flag, login and profile view is in the append-only audit log with the user and request id | done | +| "I never looked at that number" | A full identifier is shown only after `POST /audit/reveals` | partial: the API returns full numbers to reviewers (PLATFORM.md §10) | +| "The bank never sent that payment" | Only a holder of the partner's key could have signed it | partial: the transaction is stored, but not the key id, `MsgId` or signature it came with | + +### Information disclosure + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Reading data with too broad a role | `admin` sees metrics and audit, not customer data; `service` scores but cannot read alerts or cases | done | +| Customer IP addresses | Reduced to the /24 or /48 at the edge; the address is never stored, logged or returned | done | +| Numbers and e-mails in scam reports | Masked on the way out | partial: pattern-based, names in words get through | +| Scam messages a customer checks | Classified and discarded; not stored | done | +| Evidence sent to a language model | Off by default; identifiers masked before it leaves | done | +| Error messages revealing internals | One error shape; validation errors never echo the value; signature refusals never show the expected value | done | +| Secrets in the repository or the image | `.env` and `backend/secrets/` ignored by git; `*_FILE` variables read Docker/Kubernetes secrets; keyrings written with mode 0600 | done | +| Secrets in the environment of the process | `*_FILE` and a provider hook avoid it | partial: only a directory provider ships, not a vault client | +| Reasons for a decision leaking to a fraudster | `/v1/score` and callbacks return the action, not the reasons | done | +| A captured backup | — | not done: no encryption at rest beyond the disk's own | + +### Denial of service + +| Threat | Mitigation | Status | +| --- | --- | --- | +| Flooding the login or customer endpoints | Per-address and per-wallet rate limits in Redis | done | +| Flooding the stream | `503 backlog_full` above the backlog limit; batches capped at 500 | done | +| Replaying a signed ingest request | Nonce kept in Redis for twice the ±5 minute window; `TxId` deduplicated for 7 days | done | +| Large bodies | 2 MB at the proxy, 1 MB in the API | done | +| A partner's slow or dead callback endpoint | 5 s timeout, backoff, 8 attempts, then a dead list; delivery is off the request path | done | +| Redis down | Token checks fail closed (`503`); scoring needs it for the stream | partial: no Redis replica | +| One scorer process | — | see PLATFORM.md §10 | + +### Elevation of privilege + +| Threat | Mitigation | Status | +| --- | --- | --- | +| A service account reading cases | A role check on every route | done | +| An analyst approving their own freeze | Two-person rule in the database workflow | done | +| A partner key used for other partners or the console | Partner keys are accepted only on `/v1/ingest`, and JWT keys only for tokens | done | +| A leaked key staying valid | `keys retire` cuts one off at once; rotation keeps old keys only for a grace period | partial: rotation is manual, not scheduled | +| Server-side request forgery via callbacks | Callback URLs are set by operators in the keyring, `https` only in production, redirects not followed | partial: no check of where the name resolves | + +## 4. Personal data + +| Data | Kept | Shown to | +| --- | --- | --- | +| Wallet, agent and device numbers | In full, in transactions and cases | Analysts and supervisors (masked on screen until a recorded reveal) | +| Amounts, times, districts, channels | In full | Analysts and supervisors | +| Customer IP address | Only its network (/24, /48) | Nobody: used by the policy only | +| Scam-report text | As written | Reviewers, with numbers and e-mails masked | +| Scam messages checked by a customer | Not kept | — | +| Names | Not collected | — | +| Analysts' accounts | Username, role, Argon2id hash | Admins | + +Nothing personal goes to the language model unmasked, and nothing goes to it at +all unless it is switched on. + +## 5. Retention + +The policy below is what FraudLens is designed for. **Enforcing it is not built**: +there is no scheduled job that deletes or anonymises rows. The Redis entries are +the exception: they expire by themselves. + +| Data | Kept for | Then | +| --- | --- | --- | +| Transactions and decisions | 5 years (the Money Laundering Prevention Act 2012 asks for at least 5 years of transaction records) | Wallet and device numbers replaced by a keyed hash; amounts and times kept for models | +| Cases, case events, verdicts, freeze requests | 5 years after the case is closed | As above | +| Scam reports | 2 years | Text deleted, category kept | +| Audit log | 5 years | Exported and archived, then deleted | +| Redis: ingest nonces | 10 minutes | Expire | +| Redis: idempotency answers | 24 hours | Expire | +| Redis: seen `TxId`s | 7 days | Expire | +| Redis: revoked tokens | Until the token would have expired | Expire | +| Redis: callback dead list | Last 10,000 | Trimmed | +| Model training data | The synthetic dataset; no real customer data is used | — | + +A customer who asks for their data gets what is held under their wallet number; +deleting it before the period ends is refused where the law requires keeping it. + +## 6. Not done, in order of what to build next + +1. TLS on the private hops (API to Postgres and Redis), or a service mesh. +2. A secrets-manager client behind the provider hook, and scheduled rotation. +3. A retention job implementing §5. +4. Signed model and policy artifacts, checked at start-up. +5. The audit log shipped to write-once storage outside the database. +6. Encrypted backups. diff --git a/docs/plans/01-external-validation.md b/docs/plans/01-external-validation.md new file mode 100644 index 0000000..d6b8f53 --- /dev/null +++ b/docs/plans/01-external-validation.md @@ -0,0 +1,51 @@ +# Plan 01 — External validation and statistical rigour + +## Goal + +Answer the judges' common objection ("bounded by synthetic data, not validated in +the real world") with measured evidence, and put error bars on the headline numbers. + +1. Run the FraudLens modelling approach (LightGBM on receiver-centric, velocity and + cash-out-timing features; Isolation Forest as the label-free baseline) on public, + externally generated fraud data: PaySim (mobile money, simulated from real + African MFS logs) first, the Bank Account Fraud suite (NeurIPS 2022) if a public + mirror exists. Temporal split, recall at a fixed false-positive rate, PR-AUC, + and simple baselines. +2. Bootstrap 95% confidence intervals for the headline metrics on the existing + synthetic test period (scam recall at the warn operating point, false-positive + rate on legitimate payments, PR-AUC), written to a JSON artifact. +3. `make external`, and an "External validation" section in `docs/MODEL_CARD.md` + with the measured numbers. If a dataset cannot be obtained, say so. + +## Judge criteria moved + +- **AI/ML (14.67/20):** evidence that the method is not an artefact of our own + simulator, plus uncertainty quantification instead of point estimates. +- **Business impact (13.67/20):** the business case rests on the detection + numbers; showing how they hold up on independent MFS data (and how wide the + intervals are) makes the impact claim defensible. + +## Steps + +1. `scripts`/`backend/src/fraudlens/external/download.py`: fetch from public + mirrors (Hugging Face, GitHub raw, Zenodo) into `backend/data/external/` + (already gitignored via `backend/data/`), verify SHA-256 checksums. +2. `external/paysim.py`: per-step (hourly) temporal split; features computed + strictly from the past (receiver in-degree, distinct senders, receiver + velocity, time since receiver last received, sender velocity, amount ratios, + type). No balance-error leakage features in the main model (reported + separately as a known PaySim artefact). LightGBM same params; baselines: + amount rule, PaySim's own `isFlaggedFraud` rule, logistic regression, + Isolation Forest. +3. `external/baf.py` if BAF is reachable: month-based temporal split, recall at + 5% FPR (the benchmark's own metric). +4. `models/bootstrap.py`: day-block and case-cluster bootstrap of v4's test + scores; writes `artifacts/models//bootstrap_ci.json`. +5. Make target, model card section, tests on small synthetic frames. + +## Definition of done + +- `make external` downloads (or reuses) data, verifies checksums, trains, + writes `backend/artifacts/external/*.json` and the bootstrap JSON. +- Model card section quotes only numbers read from those JSON files. +- Tests and `make lint` pass; changes committed on the worktree branch. diff --git a/docs/plans/02-bangladesh-calibration.md b/docs/plans/02-bangladesh-calibration.md new file mode 100644 index 0000000..8605f31 --- /dev/null +++ b/docs/plans/02-bangladesh-calibration.md @@ -0,0 +1,42 @@ +# Plan 02: Bangladesh evidence, simulator calibration, real-user validation kit + +## Goal + +Replace "informed guesses" with sourced Bangladesh figures wherever a public +source exists, say plainly where none does, and give the team the instruments +to collect real evidence from customers and analysts. + +## Judge criteria it moves + +- **Problem relevance (16/20).** The problem is stated in sourced numbers + (MFS scale, scam reports, regulation) instead of assertion. +- **Business impact (13.67/20).** Judges said results are "bounded by synthetic + data, real-world impact not yet validated". A calibration profile tied to + Bangladesh Bank statistics, plus a ready-to-run survey and interview guide, + turns that from an open gap into a concrete validation path. + +## Steps + +1. Research (web): Bangladesh Bank monthly MFS statistics, published MFS scam + figures, Bangladesh Bank MFS regulation and customer-protection guidance, the + UK PSR APP-scam reimbursement rules. Every figure gets a URL and access date. + Nothing unsourced is presented as sourced. +2. `docs/BANGLADESH_CONTEXT.md`: problem in numbers, stakeholder map, + regulatory fit, 3–5 cited pitch openers. +3. Simulator: compare `simulator/` parameters with sourced figures. Add an + opt-in `calibrated` profile (`SimConfig.profile`, `FRAUDLENS_SIM_PROFILE` + env var, `--profile` flag). The default profile and its output are unchanged + (same seed, same bytes). Add the comparison table to `DATA_ASSUMPTIONS.md`. +4. Validation kit: `docs/validation/customer_survey.md` (Bangla + English + consent, 10–15 questions), `docs/validation/analyst_interview_guide.md`, + `backend/src/fraudlens/validation/survey_analysis.py` (CSV in, summary out) + with a test on an obviously fake fixture. + +## Definition of done + +- Every number in `BANGLADESH_CONTEXT.md` and the calibration table has a source + URL and access date, or is marked "no public source found". +- `SimConfig()` default is unchanged; a test checks the calibrated profile + differs only in the sourced parameters and that default is untouched. +- `survey_analysis.py` runs on the fixture and is tested. +- `uv run pytest -q` and `make lint` pass; committed on the worktree branch. diff --git a/docs/plans/03-business-case.md b/docs/plans/03-business-case.md new file mode 100644 index 0000000..a699f58 --- /dev/null +++ b/docs/plans/03-business-case.md @@ -0,0 +1,59 @@ +# Plan 03 — Business case in taka + +## Goal + +Turn the back-test numbers (86.9% of scams caught at warn, 0.53% of payments +falsely interrupted) into a monthly profit-and-loss a provider's CFO and CRO can +check line by line: money kept from scammers, what the friction and the review +team cost, and what is left. Every assumption is explicit, editable and has a +stated source or rationale; every model number is read from the existing +`insights.json`, so nothing is re-scored and no artifact changes. + +## Judge criterion it moves + +**Business / customer impact (13.67/20).** Today the impact page speaks in +recall and alerts a day. A judge has to do the conversion to money and headcount +themselves, and the synthetic taka figures do not scale to a real provider. This +work answers "is it worth running, at our volume, and how many people does it +need?" in BDT, and shows which assumptions the answer depends on. + +## Steps + +1. `backend/src/fraudlens/decision/business.py`: a pure model over the insights + report. + - `Assumptions` (pydantic, bounded fields) with a source or rationale per + field: volume, average payment, real scam rate, share stopped by warn / + step-up / hold, abandonment, support contacts per interruption and their + cost, analyst minutes per hold, analyst cost, coverage floor, platform + cost, reimbursement share, reputational exposure. + - The slider moves the warn threshold; step-up and hold stay where the + policy puts them (or follow the slider when it passes them). Tier bands are + differences of the cumulative sweep and `at_thresholds`. + - Scaling: rates per payment from the back-test, multiplied by monthly + volume; true alerts and scam money rescaled from the synthetic scam rate to + the assumed real one. + - Output per sweep point: prevented loss, friction cost, analyst cost, net + benefit (customers + provider, and provider only), prevented loss per ৳1 + of operating cost, analyst FTE, honest customers interrupted per 10,000. + - `sensitivity()`: tornado table, each assumption at a low and a high value, + sorted by swing in net benefit. + - CLI `python -m fraudlens.decision.business` prints the table. +2. API: `GET /v1/model/business` (defaults) and `POST /v1/model/business` + (assumption overrides) in `routes/ops.py`, next to `/model/report`. +3. Console: extend `app/(console)/impact/page.tsx` with a business-case section: + money stats that follow the slider, editable assumptions (volume presets, + analyst cost, abandonment, scam rate), and the tornado table. Types added to + `lib/types.ts`. +4. Tests: `backend/tests/test_business.py` (arithmetic on a hand-made report, + monotonicity, sensitivity ordering, real artifact, API route). +5. `docs/BUSINESS_CASE.md` with measured numbers and the tornado table, honest + about the synthetic data; one link from README. + +## Definition of done + +- `uv run pytest -q` and `make lint` pass; new tests cover the model and route. +- Dragging the impact slider updates money saved, friction cost, net benefit and + analyst headcount; editing volume, analyst cost or abandonment recomputes. +- `docs/BUSINESS_CASE.md` numbers match `python -m fraudlens.decision.business` + on the current artifacts. +- No default artifact or headline number changed; shared-file edits are additive. diff --git a/docs/plans/04-label-realism.md b/docs/plans/04-label-realism.md new file mode 100644 index 0000000..ee6898e --- /dev/null +++ b/docs/plans/04-label-realism.md @@ -0,0 +1,68 @@ +# 04 — Label realism: what under-reporting costs, and what gets it back + +## Goal + +MODEL_CARD §12 admits that every model is trained on ground-truth labels, while a +real MFS only learns about the scams victims report (about half). Turn that +disclosed weakness into a measured result: train the same transaction model under +realistic label regimes, evaluate every one on the same ground-truth test period, +and show which scam types under-reporting hides and how much of the loss a +recovery method wins back. + +## Judge criteria it moves + +- **AI/ML depth (14.67/20).** Positive-unlabelled learning, network label + propagation and a simulated review loop, compared under one protocol with + several seeds and mean ± sd: a real experiment, not one more model. +- **Business impact (13.67/20).** The judges discounted the numbers as + synthetic-label numbers. This answers the obvious objection ("would it still work + with the labels a bank actually has?") with a measured cost in scams missed, by + typology, and names the cheapest fix (analyst verdicts, mule-network labels). + +## What the simulator already models + +`simulator/fraud.py` lets each victim transfer report with probability +`report_rate = 0.5`, uniformly at random, after a lognormal delay (median 1.5 +days, 5 days for the held-out investment scam). A report only flags the mule +wallet; which cases were reported is not written to the dataset. So the reporting +draw is re-simulated per case, from data that is in the dataset. + +## Steps + +1. `backend/src/fraudlens/models/label_realism.py`, `make label-realism`. + Model: the transaction LightGBM exactly as `train.fit_booster` builds it + (same params, train fold, early stopping on val_a), seeds varied. +2. Reporting process (per case, seeded): report probability from a logistic in + log loss, victim urban/rural, victim account tenure (the simulator has no + person age, so tenure stands in for it) and typology (unauthorised takeover + reported most, shame-heavy lottery/investment scams least); intercept solved + so the mean is 50%, matching `report_rate`. Report delay as the simulator's; + labels are those known at training time (end of val_b). A reported case labels + every transaction tagged with it (victim transfers, forwards, cash-outs). + A uniform 50% regime, the simulator's own process, is kept as a check. +3. Regimes, all evaluated on ground truth: + a. ground truth (baseline); + b. report-only labels, biased; b′. report-only, uniform 50%; + c1. b + PU learning (Elkan–Noto weighting with out-of-fold propensities); + c2. b + label propagation through the mule network (wallets that received + reported scam money and their next hop pass soft labels to their other + incoming and outgoing transfers in a time window); + d. b + the analyst-verdict loop: a model trained on early report-only labels + alerts on the later training days, simulated analysts label the alerts, + and the verdicts join through `train.add_feedback`, the code path + `mlops/feedback.py` feeds. +4. Metrics on the test period: PR-AUC (all fraud-chain rows), victim-transfer + and scam recall at the served model's operating false-positive rate (v4 warn + tier, read from its `test_scores.parquet`), recall by typology. Five seeds, + mean ± sd. +5. Write `backend/artifacts/reports/label_realism.json` (new file, nothing + existing is overwritten). Add a "Under-reporting" section to MODEL_CARD with + the measured numbers. +6. Additive API field on `/model/report` and a small card on the console model page. +7. Tests on the reporting draw, propagation and PU weights on a small frame. + +## Definition of done + +- `make label-realism` reproduces the JSON; the model card quotes it, number for number. +- `uv run pytest -q` and `make lint` pass; new tests cover the label regimes. +- No served model, default artifact or headline number changes. diff --git a/docs/plans/05-adaptive-adversary.md b/docs/plans/05-adaptive-adversary.md new file mode 100644 index 0000000..3253723 --- /dev/null +++ b/docs/plans/05-adaptive-adversary.md @@ -0,0 +1,83 @@ +# Plan 05 — Adaptive adversaries + +## Goal + +`docs/DATA_ASSUMPTIONS.md` admits the weakest point of the evaluation: "Real +scammers adapt to controls; the simulator's do not react to the model's +decisions." This plan closes that gap with a measured experiment: after FraudLens +is deployed, scam cells change their scripts in rounds, each round moving toward +the evasion tactics that got past the deployed model in the previous one. We then +compare what keeps detection up: nothing (a frozen model), retraining every round +through the existing retrain recipe, or retraining only when the drift monitor +asks for it. + +## Judge criteria it moves + +- **AI/ML depth (14.67/20).** A closed-loop adversarial evaluation, retraining on + biased labels (verdicts on alerts plus victim complaints, not ground truth for + everything), drift detection measured against a non-adaptive control, and SHAP + on what carries detection after adaptation. This is the question a fraud team + actually faces after launch, answered with numbers instead of a sentence. +- **Innovation (7.67/10).** Few fraud demos model the attacker at all. A + simulator whose scam cells learn from the deployed model's decisions, and an + honest result on whether the standard drift monitor notices, is new for this + project and uncommon for hackathon fraud work. + +## Design + +- **Opt-in adversary mode** (`fraudlens.simulator.adversary`). Nothing in the + default dataset changes: the experiment runs the default world for its 120 days + (and checks it reproduces the stored dataset exactly, so the stored feature + engine snapshot can continue from it), then switches the fraud planner to an + adaptive one. New scam cells set up for a warm-up week, then run in rounds of a + week. +- **Tactics**, one drawn per scam from the cell's current mix: + `none` (the original script), `split_amounts` (victim transfers split below a + cap), `seasoned_mule` (an old recruited wallet with a normal history instead of + a young one), `delayed_cashout` (mule waits hours to days, past the 30-minute + hold-review window, before moving money), `fan_out` (each victim transfer to a + different mule from a larger pool), `mimic_hours` (scam timed at the victim's + own usual hour). +- **Adaptation**: after each round the adversary sees which of its scams got + through (no victim transfer alerted at the warn tier of that arm's deployed + model) and reweights its mix multiplicatively toward tactics with the higher + success rate, with an exploration floor. +- **Arms** (each its own simulation, because the adversary reacts to whichever + model is deployed in that arm): + - control: frozen model, adversary does not adapt (mix fixed at round 0) + - (a) frozen served model + - (b) retrained every round with the `models.train` recipe and the + `mlops.retrain` label semantics: verdicts on alerted transactions plus + complaint labels on reported scams, added to the training fold + - (c) retrained only in rounds where `mlops.drift.measure` raises an alert on + the round's traffic (the production trigger) +- **Drift**: `mlops.drift.measure` on every round's traffic against the model's + stored reference, for the adaptive and the control runs, plus the same monitor + pointed at confirmed fraud only (reference: training-period fraud). +- **SHAP**: mean |TreeSHAP| on victim transfers for the frozen model before and + after adaptation and for the last retrained model. +- **Runtime**: the four arms share nothing but the stored day-120 engine snapshot; + subsampling, if needed, is stated in the artifact and the model card. + +## Steps + +1. `Simulation.run_days` (additive; `run` calls it), `transaction_table` factored + out of `generate.build_tables` (behaviour-preserving). +2. `simulator/adversary.py`: tactics, mix update, `AdaptivePlanner`. +3. `models/adversary.py`: the experiment, artifact + `artifacts/reports/adversary.json`; `make adversary`. +4. Tests: default data unchanged by the adversary code, tactics change what they + claim to change, mix moves toward successful tactics, experiment smoke on the + small world. +5. `docs/MODEL_CARD.md` section "Adaptive adversaries" with the measured numbers; + `DATA_ASSUMPTIONS.md` updated to point at it. +6. If clean: an additive API field and a small chart on the model page. + +## Definition of done + +- `make adversary` runs end to end and writes `backend/artifacts/reports/adversary.json` + with recall per round per arm, per-tactic recall, mix per round, drift per round + and SHAP top features; no existing artifact is overwritten. +- The model card section quotes only numbers from that artifact. +- `uv run pytest -q` and `make lint` pass; new tests cover the adversary mode. +- Committed on the worktree branch. diff --git a/docs/plans/06-scaling.md b/docs/plans/06-scaling.md new file mode 100644 index 0000000..9796612 --- /dev/null +++ b/docs/plans/06-scaling.md @@ -0,0 +1,95 @@ +# Plan 06 — Horizontal scaling of the scorer + +## Goal + +Run the stream scorer as N worker processes in one Redis Streams consumer group, +with the same decisions a single process makes (0 differing feature rows against +the offline evaluation), and prove the gain with a load test on the repository's +own replay data. Ship the deployment that goes with it: Kubernetes manifests with +an autoscaler on stream lag, and a compose override that scales workers. + +## Judge criterion it moves + +**Scalability & integration (6.67/10).** The comments were "operating on a +single-process scorer" and "scaling and security hardening still pending"; +ARCHITECTURE §7 lists "horizontal scaling of the scorer" as not built. This plan +answers the scaling half with code, numbers measured on this laptop and +manifests that validate. (Security hardening is another agent's plan.) + +## What has to be shared, and why it is hard + +The feature state (`FeatureEngine`: per-wallet counters, the transfer graph, +devices, agents, flags) is in the scorer's memory, and a feature vector is only +right if every earlier state change was applied exactly once and in order. It +cannot be partitioned by wallet: a send touches two wallets, a flag walks the +graph three hops. Moving it to Redis would put a network round trip inside every +feature. + +Measured on the test period (20,000 events): computing features and applying the +event costs **17 µs** an event; the decision (three models, policy, explanation, +similar cases, case note) costs **1.3 ms** an event. Over 98% of the work does not +need the order. + +## Design + +1. **The database stays the log; every process is a replica.** Each process keeps + its own `FeatureEngine` and catches up from the log (`applied_seq` on + transactions and flags) before it changes anything, the way a restart already + rebuilds it. A party first seen by another process is registered from its + database row, so every replica holds the same state. +2. **One writer at a time, in stream order.** The ordered step (dedupe, frozen and + stale checks, registration, features, apply, insert the transaction rows) + runs under a Postgres advisory lock and only after every earlier stream entry + has been through it (a cursor in Redis). It is microseconds per event. +3. **Decisions in parallel.** For recorded events and unscored types the state + moves whatever the decision, so their feature vectors are stored in a new + `pending_decisions` table in the same commit and the decision is made after the + lock is released, by whichever worker sequenced them. Live scored events + change the state only if allowed, so they are still decided inside the lock. +4. **Exactly-once effects.** Dedupe on `txn_id` under the lock; the decision + commit claims its pending row with `DELETE … RETURNING`, so two workers that + both hold an entry write one decision. Case attachment takes a per-wallet + advisory lock (one open case per wallet already has a unique index). +5. **Reclaim.** Each worker has its own consumer name. Entries idle longer than + `claim_idle_ms` (a dead worker's) are taken with `XAUTOCLAIM` and finished; + a worker waiting on a dead predecessor reclaims it itself. +6. **Separate process.** `python -m fraudlens.platform.worker` runs one worker + with the model and policy loaded once, a small health/metrics HTTP port, and + graceful SIGTERM. The API keeps its in-process worker by default + (`FRAUDLENS_RUN_WORKER`), so `make demo` is unchanged; the API's replica + follows the log in the background. + +## Steps + +1. Plan (this file). +2. Migration `0004`: `pending_decisions`. +3. `Scorer`: incremental catch-up shared with `recover`, global/case/wallet + advisory locks, `sequence()` + `decide_pending()` split, `process()` built on + them; the workflow's `transaction()` takes the same locks. +4. `stream.Worker`: unique consumer, ordering barrier, `XAUTOCLAIM`, two-phase + batch, latency metrics. `platform/worker.py` command. +5. Tests: two workers in one group give the offline-identical decisions; a + redelivered entry and a duplicate decision commit write one decision; a dead + consumer's entries are reclaimed; a replica sees another process's changes. +6. Load test `fraudlens.platform.loadtest`: 1, 2, 4 worker processes on the same + slice of the test period, fresh database each run; throughput, p50/p95/p99 + end-to-end latency, and a parity check. Results JSON in `docs/scaling/`, table + in `docs/SCALING.md` with the hardware. +7. `deploy/k8s`: API and worker Deployments, Services, probes, limits, PDBs, KEDA + `ScaledObject` on stream length, a documented HPA alternative; validated with + `kubectl --dry-run=client`. `docker-compose.scale.yml` for `--scale worker=4`. +8. Docs: SCALING.md, PLATFORM §8/§10, ARCHITECTURE §3/§7, README pointer. +9. `uv run pytest -q` and `make lint` green; commit. + +## Definition of done + +- N workers consume one group; the replay through 2+ workers passes `verify` + with 0 tier mismatches and 0 differing feature rows (test and load test). +- Tests for consumer group, idempotency, reclaim and replica catch-up pass with + the rest of the suite against the isolated containers. +- `docs/SCALING.md` has a table of measured throughput and latency for 1, 2 and 4 + workers, the raw JSON beside it, and the hardware stated. +- `deploy/` validates with `kubectl apply --dry-run=client` (the KEDA object + with its CRD stated as a prerequisite); the compose override starts. +- Limits stated plainly: the ordered step is still one at a time, live scored + events are decided inside it, a replica's reads may lag by a poll interval. diff --git a/docs/plans/07-ingest-security.md b/docs/plans/07-ingest-security.md new file mode 100644 index 0000000..15f395e --- /dev/null +++ b/docs/plans/07-ingest-security.md @@ -0,0 +1,67 @@ +# Plan 07 — Core-banking ingest and security hardening + +## Goal + +A core-banking system or payment switch can push transactions to FraudLens over +a signed webhook, the way a bank partner would, and get a decision back (in the +response, or later by a signed callback). The service's secrets can come from +files or a secrets store, its signing keys can be rotated without logging +everyone out, it can sit behind TLS, and rate limits see the real client behind +a proxy. A STRIDE threat model says what is protected and what is not. + +## Which judge criterion it moves, and why + +- **Scalability & integration (6.67/10)**: "without live core banking webhooks". + The ingest endpoint is that webhook: HMAC-signed, replay-protected, + idempotent, ISO 20022 pacs.008-shaped, with a partner SDK and a decision + callback. It feeds the existing Redis stream, so it adds an integration path, + not a second scoring path. +- **Responsible AI & security (4/5)** and "security hardening still pending": + key rotation with `kid`, secrets from files, TLS with HSTS, a trusted-proxy + setting, and docs/SECURITY.md with every threat's mitigation and an honest + status. + +## Steps + +1. `platform/keys.py`: keyrings (JSON files) for JWT signing keys and partner + HMAC keys, re-read when the file changes; a rotation command + (`python -m fraudlens.platform.keys rotate-jwt|rotate-partner|list|retire`). +2. `platform/security.py`: tokens carry `kid`; the current key signs, any + unretired key verifies. Without a keyring the single `FRAUDLENS_JWT_SECRET` + still works as before. Production guard extended. +3. `secrets.py` + a settings source in `config.py`: `FRAUDLENS__FILE` + reads a setting from a file (Docker/Kubernetes secrets); an optional + `FRAUDLENS_SECRETS_PROVIDER=module:function` fills what is still unset. +4. `platform/ingest.py` + `api/routes/ingest.py`: + `POST /v1/ingest/transactions` (one pacs.008-style message, queued, or + decided synchronously with `?wait=decision`) and + `POST /v1/ingest/transactions/batch` (up to 500, queued). Checks, in order: + key id, timestamp within ±5 minutes, HMAC-SHA256 over + timestamp, nonce, method, path and body; nonce single use in Redis; + `Idempotency-Key` replays the stored answer; schema; then the stream. +5. `platform/callbacks.py`: optional signed decision callback per partner, from + a Redis schedule with exponential backoff and a dead list; any API process + can deliver, a claim makes each delivery happen once. +6. `api/deps.py`: `client_ip` honours `X-Forwarded-For` only from + `FRAUDLENS_TRUSTED_PROXIES`. +7. `deploy/Caddyfile` + a `tls` profile in docker-compose: HTTPS, HSTS, the API + and console behind one origin, the proxy on a fixed address the API trusts. +8. `sdk/python/fraudlens_partner.py` (standard library only) and an example. +9. Tests: bad signature, unknown key, stale timestamp, replayed nonce, + idempotent retry, conflicting idempotency key, duplicate transaction, schema, + sync decision, batch, callback with retry; keyrings, JWT rotation, secret + files, trusted proxies. +10. Docs: docs/INGEST.md (protocol, pacs.008 mapping), docs/SECURITY.md (STRIDE, + PII and retention), PLATFORM.md "not built" list updated honestly, README. + +## Definition of done + +- The ingest tests and the existing suite pass against real Postgres and Redis; + `make lint` is clean. +- A request signed by the SDK is accepted; tampered, stale, replayed and + unknown-key requests are refused with distinct error codes. +- Rotating the JWT key leaves existing sessions valid until their tokens expire; + rotating a partner key leaves the old one valid for a stated grace period. +- `docker compose --profile tls config` is valid and the Caddyfile validates. +- SECURITY.md lists every threat with done / partial / not done, and + PLATFORM.md no longer claims anything that is not built. diff --git a/docs/plans/08-customer-experience.md b/docs/plans/08-customer-experience.md new file mode 100644 index 0000000..39f2d05 --- /dev/null +++ b/docs/plans/08-customer-experience.md @@ -0,0 +1,128 @@ +# 08 · Customer experience and customer appeals + +## Goal + +Make the sender's side of FraudLens the best part of the demo, and give the +customer a way to say "this is a genuine payment" that a person answers. + +1. The phone mock (`frontend/app/(console)/phone/page.tsx`) shows every + intervention tier (allow, warn, step-up, hold) in Bangla or English, with + natural Bangla copy, a warning that names the scam pattern the decision + points to, a visible cooling-off countdown, a "what happens next" timeline, + and proper accessibility at phone width. +2. A customer can appeal a hold or a warning from the phone. An analyst or + supervisor answers it from an appeals queue in the console: approving + releases a held payment, rejecting keeps it held for the case. Every step is + audited, every appeal has a deadline, and an approved appeal becomes a + training label through the existing feedback path. + +## Judge criteria it moves + +- **Business / customer impact (13.67/20, "low customer interruption").** The + cost of APP-fraud controls is borne by honest customers who get interrupted. + This work makes each interruption shorter and clearer (the customer is told + *which* scam it looks like, how long the wait is, and what happens next) and + gives a wrongly-stopped customer recourse with a stated deadline, instead of + waiting on a helpline. Approved appeals feed back as "legitimate" labels, so + the false positives that interrupt customers are what the next model learns from. +- **Prototype quality (11.67/15).** An end-to-end customer journey in both + languages, a new reviewed workflow that reuses the case/approval/audit + patterns, a migration, tests and a browser test. +- **Responsible AI.** Contestability: an automated hold can be challenged and is + decided by a named person within an SLA; the decision and the reasoning are in + the audit log. + +## Design + +### Customer copy keyed by scam pattern + +`POST /v1/demo/pay` and `/v1/score` already return a decision summary. It gains +two additive fields: `scenario` (scam / takeover / unusual_access / cash_out) +and `cue`, one customer-safe pattern name derived from what the decision +already recorded (the rule that fired, the scenario, and the typology of the +most similar past cases): `reported_recipient`, `impersonation`, `prize`, +`investment`, `wrong_send`, `not_you`, or `generic`. No score, no reason +detail and no identifier is added. The phone turns the cue into one plain +question per language ("Did someone claiming to be from upay, bKash or your +bank ask you to send this?"). The policy's own fixed `customer_message` is +still shown unchanged. + +### Appeals + +- Table `appeals` (migration `0004_customer_appeals`, additive): one per + transaction; wallet, case, tier at the time, relation to the recipient, the + customer's reason, status (`pending` / `approved` / `rejected`), `filed_at`, + `sla_due_at`, decider, note, decided_at. +- Who can appeal: the sender of a payment whose decision was warn, step-up or + hold. A hold can be appealed only while it is still held. +- SLA: a hold's appeal is due within the hold review time from the policy + (30 minutes); a warning or step-up appeal (feedback only, the money is the + customer's to send) within 24 hours. +- Decide: analyst or supervisor. An escalated case needs a supervisor (same + rule as verdicts). Approve releases a still-held payment through the same + `Scorer.complete` the verdict uses; it refuses to release money into a wallet + that is confirmed fraud or frozen. Reject keeps the hold; the case still + decides. Approving a warning appeal changes no money: it records that the + warning was wrong. Neither shortens a step-up's cooling-off (that would be + exactly what a scammer coaching a victim would ask for). +- Audit: `appeal.file`, `appeal.approve`, `appeal.reject`; case timeline events + `appeal_filed` / `appeal_approved` / `appeal_rejected`. +- Feedback: an approved appeal labels its decision `legitimate` (y = 0) in + `mlops.feedback.labels`, unless the case it belongs to has its own verdict, + which wins. A rejected appeal is not a label: "keep it held" is not "fraud". +- Endpoints: `POST /v1/customer/transactions/{txn_id}/appeal` and + `GET .../appeal` (service); `POST /v1/demo/appeal`, `GET /v1/demo/appeal` + (staff playing the phone); `GET /v1/appeals`, `POST /v1/appeals/{id}/approve|reject` + (analyst, supervisor). + +### Console + +New page `/appeals` (nav: Investigations), modelled on Freeze approvals: status +chips, a table with SLA countdown/overdue, the customer's relation and reason +(masked), the tier, the case link, and Approve / Reject through the existing +`ReasonDialog`. + +### Phone + +- Language toggle (Bangla default, English), stored per viewer; every phone + string from one copy table; `lang` attributes set. +- Warn: pattern question, policy message, "Cancel" as the primary action, + "Send anyway" (dismissible: the policy lets the customer decide), "This is a + genuine payment" (appeal), report. +- Step-up: live mm:ss cooling-off countdown (`role=timer`), progress bar, PIN, + "Verify and send" enabled once the wait is over; cancel; appeal. +- Hold: "Your money has not left your wallet", review deadline countdown, + timeline (paused → reviewer → decision), appeal and report. +- Appeal screen: relation (radio group), reason (textarea with counter), + submit; then appeal status with its deadline and a refresh. +- Accessibility: focus moves to each new screen's heading, `aria-live` status, + labelled controls, 44px touch targets, contrast checked, phone frame fits a + 360px viewport without horizontal scroll. + +## Steps + +1. Plan (this file). +2. Backend: cue in the decision summary; `Appeal` model + migration 0004; + `platform/appeals.py` workflow; routes (customer, demo, reviewer queue); + feedback labels; tests in `test_platform.py`. +3. Console: types, `/appeals` page, nav entry. +4. Phone rewrite: language toggle, per-tier screens, countdowns, timeline, + appeal flow, accessibility. +5. Playwright: `frontend/scripts/phone-e2e.cjs` (`make e2e-phone`): pay a warn + and a hold in both languages, file an appeal, approve it in `/appeals`, see + the payment released on the phone. Add `/appeals` to the smoke pages. +6. Docs: PLATFORM.md customer section and endpoint table. +7. `uv run pytest -q`, `make lint`, e2e against a private stack + (`docker compose -p fl-cx`, ports 55434/56381, API 8028, console 3208). + +## Definition of done + +- Every phone screen reads correctly in both languages; no mixed-language + strings except brand names. +- Warn and step-up stay dismissible exactly as the policy says; hold is not. +- Appeal round trip works: file from the phone → appears in `/appeals` with an + SLA → approve releases the held payment → phone shows it sent; reject keeps + it held. All audited. +- Approved appeals appear in `/v1/feedback` and `feedback.labels()`. +- Migration matches the models (`test_the_schema_matches_the_models`). +- `pytest`, `make lint` pass; the e2e passes locally if the stack runs. diff --git a/docs/plans/09-consortium.md b/docs/plans/09-consortium.md new file mode 100644 index 0000000..d7207b3 --- /dev/null +++ b/docs/plans/09-consortium.md @@ -0,0 +1,64 @@ +# Plan 09: a privacy-preserving cross-provider mule-intelligence consortium + +## Goal + +Mule rings in Bangladesh spread their wallets over bKash, Nagad, Rocket and upay, so +no single provider sees the whole ring. Give FraudLens a protocol through which +providers share *who the mules are* without sharing *who their customers are*: + +- each provider publishes a signed bundle of blinded tokens for the wallet numbers + (MSISDNs) and handset fingerprints of its confirmed and suspected mules; +- tokens come from an oblivious PRF held by the consortium hub, so raw identifiers + never leave a provider, and a member cannot compute tokens offline to enumerate + the 01XXXXXXXXX number space; +- partners receive a Bloom filter of the tokens (local, offline check at payment + time) and fetch a listing's details only on a hit; +- every listing carries confidence, typology, first-seen, expiry, and can be + disputed and withdrawn; every share and lookup goes into a hash-chained audit log; +- the lookup becomes a new receiver-side signal. + +## Judge criteria it moves + +- **Innovation (7.67).** Cross-provider, privacy-preserving mule intelligence is not + something any one provider can build; it is the project's most distinctive idea. +- **Scalability (6.67).** The design scales across institutions, not just across + machines: per-payment checks are a local Bloom-filter probe (no network call), + bundles are deltas, and the hub only does O(listings) work. + +## Steps + +1. `backend/src/fraudlens/consortium/`: + `crypto.py` (2048-bit MODP group, OPRF with multiplicative blinding, Schnorr + signatures, Bloom filter: stdlib only), `protocol.py` (identifiers, listings, + signed bundles, partner ledger, disputes, hash-chained audit), `hub.py` (member + registry, rate-limited OPRF evaluation, bundle relay, dispute resolution), + `simulate.py` (opt-in multi-provider overlay and the experiment). +2. Simulation, opt-in, default dataset untouched: assign every wallet of the existing + dataset to one of four providers; give each wallet a synthetic MSISDN; let a + configurable share of mule recruits hold wallets at two providers on one SIM + (and the same share of ordinary customers, so false matches are measured too). + Handset sharing comes from the dataset itself. +3. Experiment on the test period: receiver-side mule-wallet recall, wallets alerted + before any victim paid, victim transfers and hours before detection, for the + mule model alone vs mule model + consortium (handset tokens, MSISDN tokens, + both), with the consortium arm's threshold set on `val_b` to the baseline's + wallet-level FPR and FPR reported on test. Write + `backend/artifacts/consortium/report.json` and the demo state. +4. API (additive): `GET /v1/consortium` (members, feeds, privacy guarantee, + measured results), `GET /v1/consortium/matches`, `GET /v1/consortium/audit`, + `POST /v1/consortium/lookup` (wallet id -> blinded lookup, audited), + `POST /v1/consortium/disputes`. +5. Console: `/consortium` page (partner feeds, matches, privacy guarantee, results), + linked from the nav and from the rings page. +6. `docs/CONSORTIUM.md`: protocol, threat model, governance, Bangladesh Bank and + data-protection alignment, measured results. +7. Tests for the crypto, protocol, disputes, audit chain, and the API route shape. + +## Definition of done + +- `uv run pytest -q` and `make lint` pass; new tests cover blinding correctness, + signature forgery rejection, Bloom lookups, TTL, disputes and audit tamper detection. +- `python -m fraudlens.consortium.simulate` writes a report with measured numbers; + the doc quotes only those numbers. +- Default artifacts, headline numbers and the served decision path are unchanged. +- Console page renders from the API; committed on the worktree branch. diff --git a/docs/plans/10-bangla-intel.md b/docs/plans/10-bangla-intel.md new file mode 100644 index 0000000..7a9131a --- /dev/null +++ b/docs/plans/10-bangla-intel.md @@ -0,0 +1,70 @@ +# Plan 10 — Bangla, Banglish and English scam-message intelligence, tied to the payment + +## Goal + +Make the scam-message classifier (`backend/src/fraudlens/intel/`) measurably +stronger in the three ways customers write (Bangla script, Banglish, English, and +the code-mixed text in between), show *which words* made it say so, and let a +flagged message raise the risk of the payment that follows it. + +## Which judge criterion it moves, and why + +- **AI/ML depth (14.67/20).** Today there is one model, trained and tested on + synthetic templates written by the same person. This adds: a public, + licensed, human-labelled Bangla/Banglish/English/code-mixed smishing corpus as + an *external* test set (the first number not measured on our own templates); a + model comparison on the same splits (current model, word TF-IDF, character + n-grams alone, multilingual sentence-transformer embeddings), with macro-F1 per + language and per typology and CPU latency; a production choice made on that + evidence, with a fallback. +- **Innovation (7.67/10).** The message-to-payment link: a payment made shortly + after the customer checked a flagged message, to the number or for the amount + in that message, is raised to a warning with that reason shown. Fraud that + starts in an SMS is caught at the moment money moves, without keeping the text. + +## Steps + +1. **Audit** the current classifier: corpus size and languages, method, measured + numbers (from `artifacts/intel/report.json`). +2. **Grow the corpus responsibly.** + - Public data: search Hugging Face and GitHub for Bangla SMS spam/scam data, + check licences, pin the revision and checksum, document provenance. + - Synthetic: new templates for the six typologies (fake agent or helpline, + prize/lottery, wrong-number send-back, job/investment, fake government aid, + OTP/PIN phishing), a code-mixed language (`mx`), more harmless look-alikes. + - A held-out script per typology that is never trained on. + - Every family tagged with its typology; provenance written in the corpus file + and in `docs/FRAUD_TAXONOMY.md`. +3. **Compare models** (`python -m fraudlens.intel.compare`, `make intel-compare`) + on the same splits: new wording of trained scripts, held-out scripts, and the + external test set. Binary macro-F1 per language and per typology, category + macro-F1, AUC, CPU latency per message. Transformer embeddings only as an + optional extra (`uv sync --extra transformer`), never a core dependency. +4. **Pick the production model** on the evidence; keep the link check (no model) + as the fallback when no classifier is loaded, as today. +5. **Cue-level explanations**: each signal carries the exact phrase(s) that + matched, as spans of the customer's own text, in either script; plus the words + that pushed the score up most (from the character n-gram weights). Additive + fields only, so the API stays backward compatible. +6. **Message-to-payment link**: when a check is flagged, remember for 30 minutes + only the numbers, wallet IDs and amounts in it (never the text). A payment by + that wallet to one of those numbers, or for one of those amounts, is raised to + at least `warn` with the reason in English and Bangla. The recipient check + says the same before the amount is typed. +7. **Docs**: `docs/FRAUD_TAXONOMY.md`, `docs/MODEL_CARD.md` with the measured + numbers only. +8. **Tests and lint**: `uv run pytest -q`, `make lint`. + +## Definition of done + +- `make intel` trains the chosen model and writes `report.json`; `make + intel-compare` writes `compare.json` with every number quoted in the docs. +- External test set results reported per variety, with source, licence and + revision recorded. +- Each of the six typologies has trained and held-out scripts and a reported + number. +- `message-check` responses keep every existing field; signals gain `phrases`, + responses gain `highlights`. +- A payment after a flagged message to its number or amount is raised to `warn` + with a reason; covered by tests. +- No invented numbers; tests and lint pass; committed on the worktree branch. diff --git a/docs/plans/11-fairness-mitigation.md b/docs/plans/11-fairness-mitigation.md new file mode 100644 index 0000000..e548cba --- /dev/null +++ b/docs/plans/11-fairness-mitigation.md @@ -0,0 +1,71 @@ +# Plan 11 — From disclosing the account-age bias to mitigating it + +## Goal + +MODEL_CARD §13 discloses that honest payments from senders whose wallet is under +30 days old are interrupted 6.0× as often as average, and payments *to* wallets +under 30 days old 16.4× (1.50% of them held). It recommends "a separate, higher +threshold for new wallets" but nothing implements it. This plan builds that +mitigation, fits it with an explicit objective, measures what it costs, and ships +it as a selectable policy version next to the unchanged v2. + +## Judge criterion moved + +**Responsible AI & security (4/5).** The judges praised the disclosure. The next +step up is acting on it with the trade-off measured: a fairness gap that is +reduced by a versioned, auditable policy change, with recall cost, confidence +intervals and a Pareto frontier, and with the human-review guarantee intact. + +## Steps + +1. **Policy schema: segment thresholds.** Add an optional `segments:` list to the + policy file. A segment is a set of `field op number` conditions (the same + comparison table the rules use, e.g. `r_age_days < 30`), the transaction types + it applies to, and threshold overrides for that segment. First matching segment + wins. Guards, enforced at load: + - segment thresholds keep `0 < warn ≤ step_up ≤ hold < 1`, so **hold stays + reachable** in every segment (no segment can switch holds off); + - segments change only the *model's* tier; rules (`raise_to`, `hard`) are + applied afterwards exactly as before, so R01/R02 still hold and nothing + lowers a hard rule; + - the `hold` tier still requires `human_review: true` (unchanged guard). + The outcome records which segment applied and the thresholds used (rule trace + and `Decision.segment`), so every decision stays auditable. The 0–100 display + score is anchored to the thresholds actually used. +2. **Softer tier for young wallets.** The segment can raise the hold threshold + more than the warn threshold, so a payment that would have been held becomes a + step-up or a warning the customer can dismiss, while the riskiest payments are + still held for a person. +3. **Fit with an explicit objective** (`decision/mitigation.py`). Thresholds are + fitted on validation (val_a + val_b), never on test. Grid of candidate + segment thresholds (multiples of the model's own); objective: minimise the + larger of the two young-segment false-alert ratios, subject to the overall + victim-transfer recall at warn falling by at most X percentage points and the + hold-tier recall by at most Y points. Every candidate's (gap, recall) is kept, + and the Pareto frontier is reported. +4. **Evaluate on the test period, before (v2) vs after (v3):** false-alert rate + and ratio per age segment (sender and receiver under 30 days), overall false + alert and false hold rate, victim-transfer and case recall, scam taka caught, + held rate. Row-level bootstrap 95% CIs for the after − before differences. + Write `artifacts/models//fairness_mitigation.json`. +5. **Ship `policies/v3.yaml`** = v2 + fitted segments. v1 and v2 unchanged, + default stays v2 (`FRAUDLENS_POLICY_VERSION=v3` selects it). +6. **API + console:** `/v1/model/report` gains an additive `mitigation` field; + the fairness page shows the before/after table and the frontier with existing + components. +7. **Intersectional view (cheap):** account age × channel × urban/rural table in + the fairness report. +8. **Docs:** MODEL_CARD §13 and DECISION_POLICY.md with the measured numbers and + an honest trade-off statement. Tests for schema guards, segment selection, hard + rules unaffected, and the fitting objective. + +## Definition of done + +- `policies/v3.yaml` loads; a segment that removes hold (hold ≥ 1) or breaks the + threshold order is refused at load; hard rules still hold in every segment + (tests). +- `fairness_mitigation.json` exists with before/after, CIs and the frontier; every + number in the docs is read from it. +- Fairness page shows before/after and the frontier; `make lint` and + `uv run pytest -q` pass. +- Recommendation on whether v3 should become the default, stated with its cost. diff --git a/docs/scaling/loadtest-20261007-094657.json b/docs/scaling/loadtest-20261007-094657.json new file mode 100644 index 0000000..377da5d --- /dev/null +++ b/docs/scaling/loadtest-20261007-094657.json @@ -0,0 +1,203 @@ +{ + "generated_at": "2026-10-07T09:46:57+0600", + "machine": { + "platform": "macOS-27.0-arm64-arm-64bit", + "processor": "arm", + "cpu_count": 10, + "load_average_at_end": [ + 26.2, + 22.5, + 21.2 + ] + }, + "dataset": "full", + "events": 29000, + "settings": { + "workers": [ + 1, + 2, + 4 + ], + "saturation": 20000, + "rate": 200.0, + "rate_events": 9000, + "out": "/Users/mdabirhossain/Documents/WebDevelopment/FraudLens/.claude/worktrees/agent-aa249e777a4cf60ae/docs/scaling" + }, + "results": [ + { + "workers": 1, + "startup_seconds": 2.6, + "saturation": { + "events": 20000, + "seconds": 59.79, + "events_per_second": 334.5 + }, + "open_loop": { + "events": 9000, + "rate_per_second": 200.0, + "latency_ms": { + "p50": 54.1, + "p95": 439.3, + "p99": 1014.4, + "max": 1456.0, + "samples": 9000 + } + }, + "per_worker": [ + { + "consumer": "loadtest-1-0", + "processed": 29000, + "decided": 14419, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 16557.64, + "p95": 54781.77, + "p99": 58793.14 + } + } + ], + "parity": { + "ok": true, + "compared": 14419, + "tier_mismatches": 0, + "feature_rows_differing": 0 + } + }, + { + "workers": 2, + "startup_seconds": 5.2, + "saturation": { + "events": 20000, + "seconds": 34.55, + "events_per_second": 578.8 + }, + "open_loop": { + "events": 9000, + "rate_per_second": 200.0, + "latency_ms": { + "p50": 40.5, + "p95": 194.1, + "p99": 291.7, + "max": 397.8, + "samples": 9000 + } + }, + "per_worker": [ + { + "consumer": "loadtest-2-0", + "processed": 14540, + "decided": 7204, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 11220.9, + "p95": 31461.54, + "p99": 34079.87 + } + }, + { + "consumer": "loadtest-2-1", + "processed": 14460, + "decided": 7215, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 11066.97, + "p95": 30853.54, + "p99": 34043.07 + } + } + ], + "parity": { + "ok": true, + "compared": 14419, + "tier_mismatches": 0, + "feature_rows_differing": 0 + } + }, + { + "workers": 4, + "startup_seconds": 3.7, + "saturation": { + "events": 20000, + "seconds": 21.48, + "events_per_second": 931.2 + }, + "open_loop": { + "events": 9000, + "rate_per_second": 200.0, + "latency_ms": { + "p50": 105.6, + "p95": 192.2, + "p99": 287.7, + "max": 482.4, + "samples": 9000 + } + }, + "per_worker": [ + { + "consumer": "loadtest-4-0", + "processed": 7173, + "decided": 3544, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 6326.51, + "p95": 19934.48, + "p99": 20683.13 + } + }, + { + "consumer": "loadtest-4-1", + "processed": 7346, + "decided": 3626, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 5052.11, + "p95": 20271.02, + "p99": 21056.89 + } + }, + { + "consumer": "loadtest-4-2", + "processed": 6951, + "decided": 3465, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 5534.02, + "p95": 19751.9, + "p99": 20720.64 + } + }, + { + "consumer": "loadtest-4-3", + "processed": 7530, + "decided": 3784, + "dead_lettered": 0, + "failures": 0, + "reclaimed": 0, + "latency_ms": { + "p50": 5786.23, + "p95": 20541.42, + "p99": 21236.76 + } + } + ], + "parity": { + "ok": true, + "compared": 14419, + "tier_mismatches": 0, + "feature_rows_differing": 0 + } + } + ] +} \ No newline at end of file diff --git a/docs/validation/analyst_interview_guide.md b/docs/validation/analyst_interview_guide.md new file mode 100644 index 0000000..91427e8 --- /dev/null +++ b/docs/validation/analyst_interview_guide.md @@ -0,0 +1,95 @@ +# Analyst interview guide + +**Purpose.** Check FraudLens's assumptions against the people who would run it: +fraud, risk and AML analysts at an MFS provider or bank, and customer-care +staff who take scam calls. The simulator's typologies, report rates and +response times are assumptions (docs/DATA_ASSUMPTIONS.md); this interview is how +we find out which are wrong. + +**Format.** 45 minutes, one interviewee, two interviewers (one asks, one +notes). Remote or in person. No recording unless the interviewee agrees in +writing. + +**Ground rules (say these at the start).** + +- We want your professional view, not your employer's data. Please do not share + customer details, transaction records, internal thresholds or anything under + NDA. Ranges and "roughly" are fine. +- We will not name you or your organisation. Notes are kept by the team only + and summarised across interviews. +- You can skip any question or stop at any time. + +Note the interviewee's role, years in the job and type of organisation (MFS, +bank, PSP, regulator, other) only — no names. + +--- + +## 1. Today's process (10 min) + +1. Walk us through what happens from the moment a customer reports a scam to + the case being closed. Who touches it, and how long does each step take? +2. How do you find out about a scam other than a customer report? (Rules, + agent tips, other providers, police, BFIU.) +3. What rules or models run on transactions today, and do they act before the + money moves or after? +4. Roughly what share of scam reports end with any money recovered? What stops + recovery most often? + +*Compare with:* BB reported 8.7% recovery of MFS fraud value in 2025; TIB found +58.8% of personal holders who had a problem did not complain +(docs/BANGLADESH_CONTEXT.md). + +## 2. Typologies and mules (10 min) + +5. Which scam types do you see most? Put these in order: impersonation (company + or official), prize/lottery/job fee, "sent by mistake", investment scheme, + account takeover (PIN/OTP stolen). What is missing? +6. How quickly does a receiving wallet empty out after a scam payment? Cash-out + at an agent, onward transfers, or both? +7. How do mule wallets usually look: new accounts, bought accounts, agents' + own wallets? How long do they stay active before being blocked? +8. How often does the same mule or the same handset show up across providers? + Would a shared blacklist help, and what stops it today? + +*Compare with:* simulator typology mix, mule dwell time and cash-out delay +(DATA_ASSUMPTIONS.md §5) — all currently unsourced. + +## 3. The FraudLens console (15 min) + +Show the queue, a case page with its reasons, and the network view (make demo, +docs/DEMO_SCRIPT.md). Then ask: + +9. Would the reasons shown on a case let you decide faster? What is missing + that you would look up elsewhere? +10. How many alerts can one analyst clear in a shift? At what false-alarm rate + does a queue stop being worked? +11. Which of warn / step-up / hold could your organisation actually deploy + today? What would legal, compliance or customer care object to? +12. A hold delays a payment by up to 30 minutes. Who would have to approve + that, and what would make them say yes? + +*Compare with:* DECISION_POLICY.md bands and the analyst capacity assumed in +the review simulator. + +## 4. Data and deployment (5 min) + +13. Which of these signals could you get in real time: device ID, SIM-change + date, wallet age, receiver's inflow in the last hour, agent location? +14. Where would a system like this have to run (on-premise, in-country cloud), + and who signs off on a new model? + +## Close (5 min) + +15. If you could change one thing about how scams are handled, what would it + be? +16. Is there someone else we should talk to? + +Thank them. Within 24 hours, write up notes under the headings above, with +quotes marked as quotes and no identifying details. + +## After 5+ interviews + +Tally the answers to Q5, Q6, Q7 and Q10 in a table and compare with the +simulator's assumptions. Where most analysts disagree with an assumption, +change it in the calibrated profile (`SimConfig.calibrated`) and record the +source as "analyst interviews, n = X, " in DATA_ASSUMPTIONS.md §11. diff --git a/docs/validation/customer_survey.md b/docs/validation/customer_survey.md new file mode 100644 index 0000000..233bb65 --- /dev/null +++ b/docs/validation/customer_survey.md @@ -0,0 +1,134 @@ +# Customer survey: scam experience and reactions to FraudLens screens + +**Purpose.** Every FraudLens number so far comes from synthetic data. This survey +asks real MFS users two things: what scams they have actually met, and whether +they would accept the three interventions the decision policy uses (warn, +step-up, hold; docs/DECISION_POLICY.md). It takes about 8 minutes. + +**Who.** Adult (18+) personal MFS account holders. Aim for at least 100 +respondents, with both urban and rural areas and at least 30% women; TIB's 2025 +survey (1,784 respondents in 32 districts) is the benchmark to compare against. + +**How.** Face to face or by phone, by an enumerator who reads the questions in +Bangla. Record answers on the CSV template at the end; the column names are +the IDs `fraudlens.validation.survey_analysis` reads. + +**What we never ask or record.** Name, phone number, wallet number, PIN, OTP, +NID, or the exact amount in the respondent's account. If a respondent starts to +read out a PIN or OTP, stop them. + +--- + +## Consent (read aloud before any question) + +**বাংলা** + +> আসসালামু আলাইকুম। আমরা FraudLens নামে একটি ছাত্র প্রকল্পের পক্ষ থেকে এসেছি। +> আমরা মোবাইল ব্যাংকিং (যেমন বিকাশ, নগদ, রকেট) প্রতারণা কমানোর একটি পদ্ধতি নিয়ে +> কাজ করছি, এবং জানতে চাই প্রতারণার বিষয়ে আপনার অভিজ্ঞতা কী এবং কিছু সতর্কবার্তা +> আপনার কাছে কেমন লাগে। এতে প্রায় ৮ মিনিট লাগবে। +> +> আমরা আপনার নাম, ফোন নম্বর, অ্যাকাউন্ট নম্বর, পিন বা ওটিপি জানতে চাইব না — দয়া +> করে এগুলো বলবেন না। আপনার উত্তর বেনামে রাখা হবে এবং শুধু গবেষণার জন্য সামগ্রিক +> হিসাবে ব্যবহার করা হবে। আমরা কোনো এমএফএস কোম্পানি বা বাংলাদেশ ব্যাংকের প্রতিনিধি +> নই, এবং এই জরিপ কোনো হারানো টাকা ফেরত আনতে পারবে না। +> +> অংশ নেওয়া সম্পূর্ণ আপনার ইচ্ছা। যে কোনো প্রশ্ন বাদ দিতে পারেন, যে কোনো সময় থামতে +> পারেন। আপনি কি অংশ নিতে রাজি আছেন? + +**English** + +> Hello. We are from FraudLens, a student project working on a way to reduce +> mobile money (bKash, Nagad, Rocket and others) fraud. We would like to hear +> about your experience of scams and how some warning screens feel to you. It +> takes about 8 minutes. +> +> We will not ask for your name, phone number, account number, PIN or OTP — +> please do not tell us these. Your answers are anonymous and are used only in +> aggregate for research. We do not represent any MFS provider or Bangladesh +> Bank, and this survey cannot recover lost money. +> +> Taking part is entirely voluntary. You can skip any question or stop at any +> time. Do you agree to take part? + +Record `consent` = `yes` or `no`. If `no`, thank them and stop; the analysis +drops the row anyway. + +If the respondent has lost money: tell them they can complain to their +provider's helpline (providers must resolve disputes within 10 working days, +MFS Regulations 2022 §17.3) and to Bangladesh Bank's Customer Interests +Protection Centre (CIPC). + +--- + +## Questions + +Profile (enumerator fills in): `respondent_id` (e.g. `DHK-014`; never a phone +number), `area` = `urban` / `rural`. + +### Part A — scam experience (last 12 months) + +| ID | Question (English; read in Bangla) | Answers | +| --- | --- | --- | +| `q01_contacted` | Has anyone called, texted or messaged you trying to get money or a PIN/OTP from your mobile wallet? | `yes` / `no` | +| `q02_lost_money` | Did you lose money to a scam of this kind? | `yes` / `no` | +| `q03_scam_type` | *If yes:* Which is closest to what happened? (read options) | `impersonation` (posed as company, official, relative) / `lottery_fee` (prize, job or loan with a fee) / `wrong_send` ("sent by mistake, please return") / `investment` (high-return scheme) / `account_takeover` (someone got into your account) / `other` | +| `q04_loss_bdt` | *If yes:* Roughly how much did you lose, in taka? | number | +| `q05_reported_to` | *If yes:* Who did you tell? | `nobody` / `provider` / `police` / `cipc` / `several` | +| `q06_recovered` | *If yes:* Did you get any of it back? | `none` / `partial` / `full` | + +### Part B — reactions to the three screens + +Show each mock-up (screenshots from the FraudLens console, docs/screenshots) or +read the script below. Answers 1–5: 1 = strongly disagree, 5 = strongly agree. + +- **Warn.** *"Before you send Tk 8,000: this number was opened 2 days ago and + has received money from many people today. People are often told to send + money to 'return a mistake' or 'unlock a prize'. Do you know this person?"* + [Send anyway] [Cancel] +- **Step-up.** *"This payment looks like a common scam, so it is paused for 30 + minutes. Enter your PIN again to confirm it is really you; after the pause it + goes through unless you cancel."* +- **Hold.** *"We are holding this Tk 15,000 transfer for up to 30 minutes + so a person can check it. You can cancel now, or call your provider's + helpline. The money is still in your wallet."* + +| ID | Statement | Answers | +| --- | --- | --- | +| `q07_warn_would_stop` | The warning would make me stop and check before sending. | 1–5 | +| `q08_step_up_ok` | Confirming with my PIN again and waiting 30 minutes would be acceptable. | 1–5 | +| `q09_hold_ok` | Having a payment held for a while would be acceptable if it protects me. | 1–5 | +| `q10_max_hold_minutes` | What is the longest you would accept a payment being held? (minutes; 0 = never) | number | +| `q11_false_alarms_per_year` | How many times a year could a *normal* payment of yours be warned or held before you would stop trusting it? | number | +| `q12_trust` | If your wallet showed screens like these, would you trust it more, the same, or less? | `more` / `same` / `less` | +| `q13_language` | Which language should these screens be in? | `bangla` / `english` / `both` | +| `q14_comments` | Anything else you want to tell us? (free text; not analysed by the script) | text | + +--- + +## CSV template + +``` +respondent_id,consent,area,q01_contacted,q02_lost_money,q03_scam_type,q04_loss_bdt,q05_reported_to,q06_recovered,q07_warn_would_stop,q08_step_up_ok,q09_hold_ok,q10_max_hold_minutes,q11_false_alarms_per_year,q12_trust,q13_language,q14_comments +``` + +Leave a cell empty when a question was skipped or does not apply. Then: + +``` +cd backend +uv run python -m fraudlens.validation.survey_analysis responses.csv --by area +``` + +The output is JSON: shares for each categorical answer, loss median/mean/p90 +for victims only, and for each screen the mean score and the share who agree +(4–5) or disagree (1–2). + +## What the answers would change + +| Finding | Change in FraudLens | +| --- | --- | +| `q07` agree < 50% | Rewrite the warn text; test the scam-script wording against the message-check copy. | +| `q09` agree < 50% or `q10` median < 30 | Shorten the hold window in DECISION_POLICY.md or reserve hold for the top risk band only. | +| `q11` median low (≤ 2) | Tighten the warn threshold; the policy's false-positive budget is set too loose. | +| `q13` mostly `bangla` | Bangla-first copy for every customer-facing screen. | +| `q03` mix differs from the simulator's typology mix | Re-weight typologies in the calibrated simulator profile (DATA_ASSUMPTIONS.md §11). | diff --git a/frontend/app/(console)/appeals/page.tsx b/frontend/app/(console)/appeals/page.tsx new file mode 100644 index 0000000..c6cc30a --- /dev/null +++ b/frontend/app/(console)/appeals/page.tsx @@ -0,0 +1,140 @@ +"use client"; + +import Link from "next/link"; +import { useState } from "react"; + +import { Id, useSession } from "@/components/session"; +import { Async, Badge, Button, Card, Chip, Empty, PageHeader, ReasonDialog, StatusBadge, Table, Td, TierBadge } from "@/components/ui"; +import { api, useApi } from "@/lib/api"; +import { taka, when, words } from "@/lib/format"; +import type { Appeal, AppealQueue, AppealStatus } from "@/lib/types"; + +const STATUSES: AppealStatus[] = ["pending", "approved", "rejected"]; +const OUTCOME_TONE = { pending: "amber", approved: "green", rejected: "slate" } as const; + +/** Time left until the deadline, or how late it is: "12 min left", "1 h 5 min late". */ +function due(appeal: Appeal, now: string): { text: string; late: boolean } { + const minutes = Math.round((Date.parse(appeal.sla_due_at) - Date.parse(now)) / 60_000); + const span = Math.abs(minutes); + const text = span >= 60 ? `${Math.floor(span / 60)} h ${span % 60} min` : `${span} min`; + return minutes >= 0 ? { text: `${text} left`, late: false } : { text: `${text} late`, late: true }; +} + +interface Pending { + appeal: Appeal; + approve: boolean; +} + +export default function AppealsPage() { + const { canReview } = useSession(); + const [status, setStatus] = useState("pending"); + const queue = useApi(`/v1/appeals?status=${status}&limit=200`); + const [dialog, setDialog] = useState(null); + const [outcome, setOutcome] = useState(null); + + async function decide({ appeal, approve }: Pending, note: string) { + const done = await api<{ released: boolean; txn_status: string }>( + `/v1/appeals/${appeal.id}/${approve ? "approve" : "reject"}`, + { note }, + ); + setOutcome( + `Appeal #${appeal.id} ${approve ? "approved" : "rejected"}. ` + + (done.released ? "The payment was released to the recipient." : `The payment is ${words(done.txn_status)}.`), + ); + queue.reload(); + } + + return ( + <> + + {outcome &&
{outcome}
} + + +
+ {STATUSES.map((value) => ( + setStatus(value)}> + {words(value)} + {queue.data ? ` (${queue.data.counts[value]})` : ""} + + ))} +
+ + {({ appeals, now }) => + appeals.length ? ( + + {appeals.map((appeal) => { + const deadline = due(appeal, now); + return ( + + + + + + + + + + ); + })} +
+ #{appeal.id} +
{when(appeal.filed_at)}
+ 		+ {taka(appeal.amount)} +
+ → {appeal.receiver_id ? : "–"} +
+ {appeal.txn_status &&
} + 		+ {words(appeal.relation)} +
{appeal.reason}
+ 		+ {appeal.status === "pending" ? ( + {deadline.text} + ) : ( + {when(appeal.sla_due_at)} + )} + + {appeal.case_id ? #{appeal.case_id} : –} + + {appeal.status !== "pending" ? ( + + {words(appeal.status)} {appeal.decider}: {appeal.decision_note} + + ) : canReview ? ( + + + + + ) : null} +
+ ) : ( + No {status} appeals. + ) + } + + + + {dialog && ( + setDialog(null)} + onSubmit={(note) => decide(dialog, note)} + /> + )} + + ); +} diff --git a/frontend/app/(console)/consortium/page.tsx b/frontend/app/(console)/consortium/page.tsx new file mode 100644 index 0000000..35dec96 --- /dev/null +++ b/frontend/app/(console)/consortium/page.tsx @@ -0,0 +1,303 @@ +"use client"; + +import { useState } from "react"; + +import { useSession } from "@/components/session"; +import { Async, Badge, Button, Card, Empty, Facts, PageHeader, Stat, Table, Tabs, Td, inputClass } from "@/components/ui"; +import { api, useApi } from "@/lib/api"; +import { maskId, num, pct, taka, when, words } from "@/lib/format"; +import type { ConsortiumAuditEntry, ConsortiumLookup, ConsortiumMatch, ConsortiumOverview, SeedStat } from "@/lib/types"; + +type Tab = "feeds" | "matches" | "results" | "audit"; + +const ARM_LABEL: Record = { + mule_model_only: "Mule model alone (today)", + device: "+ partner handsets, tuned on val_b", + msisdn: "+ partner wallet numbers, tuned on val_b", + msisdn_and_device: "+ both, tuned on val_b", + msisdn_confirmed_rule: "+ rule: partner-confirmed number", + msisdn_and_device_confirmed_rule: "+ rule: partner-confirmed number or handset", +}; + +function mean(s: SeedStat | undefined, digits = 0, asPct = false): string { + if (!s) return "–"; + const fmt = (v: number) => (asPct ? pct(v, digits) : num(v, digits)); + return s.min === s.max ? fmt(s.mean) : `${fmt(s.mean)} (${fmt(s.min)}–${fmt(s.max)})`; +} + +export default function ConsortiumPage() { + const overview = useApi("/v1/consortium"); + const [tab, setTab] = useState("feeds"); + return ( + <> + + + {(o) => ( +
+
+ + a + f.confirmed_listings, 0))} sub="as tokens, signed per provider" /> + a + f.suspected_in_filter, 0))} sub="testable, not listable" /> + +
+ + tabs={[ + { id: "feeds", label: "Partner feeds" }, + { id: "matches", label: "Matches" }, + { id: "results", label: "Measured effect" }, + { id: "audit", label: "Audit and disputes" }, + ]} + value={tab} + onChange={setTab} + /> + {tab === "feeds" && } + {tab === "matches" && } + {tab === "results" && } + {tab === "audit" && } +
+ )} + + + ); +} + +function Feeds({ o }: { o: ConsortiumOverview }) { + return ( + <> + + + {o.feeds.map((f) => ( + + + + + + + + + + + + ))} +
{f.display}#{f.seq}{num(f.confirmed_listings)}{Object.entries(f.by_kind).map(([k, v]) => `${words(k)} ${v}`).join(", ") || "–"}{num(f.suspected_in_filter)} in {num(f.filter_bits)} bits{num(f.withdrawn)}{num(f.bytes / 1024, 1)} KB{f.signature_ok ? valid : invalid}{when(f.valid_until)}
+ + +
    + {o.guarantees.map((g) => ( +
  • {g}
    • + ))} +
+

OPRF key fingerprint {o.oprf_key_fingerprint}. See docs/CONSORTIUM.md for the threat model.

+ + + + ); +} + +function LookupCard() { + const [wallet, setWallet] = useState(""); + const [out, setOut] = useState(null); + const [error, setError] = useState(null); + const run = async () => { + setError(null); + try { + setOut(await api("/v1/consortium/lookup", { wallet_id: wallet.trim() })); + } catch (e) { + setOut(null); + setError(e instanceof Error ? e.message : String(e)); + } + }; + return ( + +
+ setWallet(e.target.value)} /> + +
+ {error &&

{error}

} + {out && ( +
+ + {out.matches.length ? ( +
    + {out.matches.map((m, i) => ( +
  • + {m.status} {m.provider}, {words(m.kind)}, confidence {num(m.confidence, 3)} + {m.listing_id && {m.listing_id} · {words(m.typology)}} +
    • + ))} +
+ ) : ( +

No partner knows this wallet.

+ )} +
+ )} + + ); +} + +function Matches() { + const matches = useApi("/v1/consortium/matches?limit=300"); + return ( + + + {(rows) => + rows.length ? ( + + {rows.map((m) => ( + + + + + + + + + + + + + ))} +
{maskId(m.wallet_id)}{when(m.at)}{m.home}{m.partners.join(", ")}{m.kinds.map(words).join(", ")}{m.status}{num(m.confidence, 3)}{num(m.mule_score, 3)}{m.mule_model_alone_would_alert ? "yes" : no: consortium only}{m.simulation_truth}
+ ) : ( + No receiver matched a partner listing in the test period. + ) + } + + + ); +} + +function Results({ o }: { o: ConsortiumOverview }) { + const r = o.results; + const head = r.results_by_shared_sim_rate[r.headline_share.toFixed(2)]?.summary ?? {}; + const arms = Object.keys(head); + return ( + <> + + + {arms.map((a) => ( + + + + + + + + + + ))} +
{ARM_LABEL[a] ?? a}{mean(head[a]?.recall, 1, true)}{mean(head[a]?.false_alerts, 1)}{mean(head[a]?.fpr, 2, true)}{mean(head[a]?.mules_found_only_with_consortium, 1)}{mean(head[a]?.["timing.detected_before_any_victim_paid"], 1)}{head[a]?.["timing.taka_paid_to_detected_mules_before_detection"] ? taka(head[a]?.["timing.taka_paid_to_detected_mules_before_detection"]?.mean) : "–"}
+ + + + {Object.keys(r.poisoning.clean).map((a) => ( + + + + + + + ))} +
{ARM_LABEL[a] ?? a}{num(r.poisoning.clean[a].false_alerts)}{num(r.poisoning.attacked[a].false_alerts)}{pct(r.poisoning.attacked[a].recall, 1)}
+ + + + + + ); +} + +function AuditTab({ o, reload }: { o: ConsortiumOverview; reload: () => void }) { + const { me } = useSession(); + const entries = useApi("/v1/consortium/audit?limit=150"); + const [listing, setListing] = useState(""); + const [reason, setReason] = useState(""); + const [error, setError] = useState(null); + const raisedBy = o.members[0]?.name ?? ""; + const act = async (path: string, body: unknown) => { + setError(null); + try { + await api(path, body); + reload(); + entries.reload(); + } catch (e) { + setError(e instanceof Error ? e.message : String(e)); + } + }; + return ( + <> + +
+ setListing(e.target.value)} /> + setReason(e.target.value)} /> + +
+ {error &&

{error}

} + {o.disputes.length ? ( +
+ + {o.disputes.map((d) => ( + + + + + + + + + + ))} +
{d.dispute_id}{d.listing_id}{d.raised_by}{d.reason}{when(d.opened_at)}{d.status} + {d.status === "open" && me.role === "supervisor" && ( + + + + + )} +
+
+ ) : ( +

No disputes yet.

+ )} + + + + {(rows) => ( + + {rows.map((e) => ( + + + + + + + + ))} +
{e.n}{when(e.at)}{e.event}{JSON.stringify(e.detail)}{e.hash.slice(0, 12)}
+ )} + + + + ); +} diff --git a/frontend/app/(console)/fairness/page.tsx b/frontend/app/(console)/fairness/page.tsx index dc54db4..96d2e58 100644 --- a/frontend/app/(console)/fairness/page.tsx +++ b/frontend/app/(console)/fairness/page.tsx @@ -2,11 +2,11 @@ import { useState } from "react"; -import { HBars } from "@/components/charts"; +import { HBars, Legend, LineChart } from "@/components/charts"; import { Async, Badge, Card, Chip, Empty, PageHeader, Stat, Table, Td } from "@/components/ui"; import { useApi } from "@/lib/api"; import { num, pct, words } from "@/lib/format"; -import type { FairRow, Insights, Report } from "@/lib/types"; +import type { FairRow, Insights, Mitigation as MitigationReport, MitigationMetric, Report } from "@/lib/types"; const DIMENSION: Record = { region: "Region", @@ -60,7 +60,130 @@ function Dimension({ name, rows, overall }: { name: string; rows: FairRow[]; ove ); } -function Fairness({ fairness, version }: { fairness: Insights["fairness"]; version: string }) { +const isRatio = (metric: MitigationMetric) => metric.key.includes("ratio"); + +function shown(metric: MitigationMetric, v: number | null): string { + return isRatio(metric) ? `${num(v, 2)}×` : pct(v, 2); +} + +/** A 95% interval of a change: ratios as they are, rates in percentage points. */ +function interval(metric: MitigationMetric, [lo, hi]: [number | null, number | null]): string { + const f = (v: number | null) => (v === null ? "–" : isRatio(metric) ? num(v, 2) : `${(v * 100).toFixed(2)} pp`); + return `${f(lo)} to ${f(hi)}`; +} + +const YOUNG = [ + { key: "receiver_young_ratio", name: "Wallet receiving, under 30 days", color: "var(--color-warn)" }, + { key: "sender_young_ratio", name: "Customer sending, under 30 days", color: "var(--color-info)" }, +]; + +/** Policy v3: young wallets get cut-offs of their own. What it changed on the test period, and what closing the gap further would cost. */ +function Mitigation({ m, side }: { m: MitigationReport; side: "sender" | "receiver" }) { + const metric = (key: string) => m.before_after.metrics.find((row) => row.key === key); + const headline = [ + { key: "receiver_young_ratio", label: "Wallet receiving, under 30 days", sub: "honest payments interrupted, against the overall rate" }, + { key: "sender_young_ratio", label: "Customer sending, under 30 days", sub: "the same, by who sends" }, + { key: "victim_recall", label: "Victim transfers alerted", sub: "warned, checked again or held" }, + { key: "taka_recall", label: "Victims’ money alerted", sub: "share of the taka scammed" }, + ]; + const path = m.fit.path_warn; + const warn = (point: (typeof path)[number]) => Object.values(point.scales)[0].warn; + const limit = path.findIndex((point, i) => point.admissible && !path[i + 1]?.admissible); + const series = YOUNG.map((y) => ({ ...y, values: path.map((point) => point.test[y.key]) })); + return ( + <> +

Narrowing the gap: policy {m.policy_version}

+

+ Payments to or from a wallet under 30 days old are scored against cut-offs of their own, fitted on the validation period and measured + here on the {num(m.days)}-day test period ({num(m.rows.test)} payments) against {m.base_policy}. The rules, including the hard ones, are + unchanged, and a held payment still waits for a person. +

+
+ {headline.map((h) => { + const row = metric(h.key); + if (!row) return null; + const ratio = isRatio(row); + return ( + = NOTABLE ? "warn" : "good") : "plain"} + sub={`${h.sub} · change ${interval(row, row.difference_ci)} (95%)`} + /> + ); + })} +
+ +
+ + + {m.before_after.metrics.map((row) => ( + + + + + + + ))} +
{row.label}{shown(row, row.before)}{shown(row, row.after)}{interval(row, row.difference_ci)}
+ + + `×${num(warn(point), 1)}`)} + format={(v) => `${num(v, 0)}×`} + xTitle="Warn cut-off, times the model’s (test period)" + mark={{ index: limit, label: "guard" }} + /> +
+ + {path.map((point) => ( + + + + + + + + + ))} +
×{num(warn(point), 1)}{num(point.test.receiver_young_ratio, 2)}×{num(point.test.sender_young_ratio, 2)}×{pct(point.test.victim_recall, 2)}{pct(point.test.taka_recall, 2)}{point.admissible ? null : past the guard}
+ +
+ + + + {m.intersectional[side].map((row) => ( + + + + + + + + ))} +
{row.group}{row.too_small ? " (too few to judge)" : ""}{num(row.legitimate)}{pct(row.false_alert_rate, 2)} → {pct(row.false_alert_rate_after, 2)}= NOTABLE ? "text-warn" : undefined}> + {num(row.ratio_to_overall, 1)}× → {num(row.ratio_to_overall_after, 1)}× + {row.victim_transfers ? `${pct(row.victim_transfers_alerted, 0)} → ${pct(row.victim_transfers_alerted_after, 0)}` : "–"}
+ + + ); +} + +function Fairness({ fairness, version, mitigation }: { fairness: Insights["fairness"]; version: string; mitigation?: MitigationReport | null }) { const [side, setSide] = useState<"sender" | "receiver">("sender"); const groups = fairness[side]; const order = [...Object.keys(DIMENSION).filter((name) => name in groups), ...Object.keys(groups).filter((name) => !(name in DIMENSION))]; @@ -93,6 +216,8 @@ function Fairness({ fairness, version }: { fairness: Insights["fairness"]; versi {order.map((name) => )} + {mitigation && } +
  • @@ -113,6 +238,12 @@ function Fairness({ fairness, version }: { fairness: Insights["fairness"]; versi
  • “Fraud caught” is the other half: an even false-alert rate is no use if one group’s victims are protected less.
    • + {mitigation && ( +
  • + Policy {mitigation.policy_version} trades a little detection for a fairer spread, and the numbers above say how much. Its cut-offs + can only narrow the gap so far: much of what remains comes from rules and from honest payments the model scores very high. +
    • + )}
  • The customers are simulated, so this shows the method and where the gaps open, not how real customers would be treated.
 @@ -131,7 +262,7 @@ export default function FairnessPage() { {(data) => data.insights?.fairness ? ( - + ) : ( This model version was evaluated without a fairness breakdown. ) diff --git a/frontend/app/(console)/impact/business.tsx b/frontend/app/(console)/impact/business.tsx new file mode 100644 index 0000000..6d9df96 --- /dev/null +++ b/frontend/app/(console)/impact/business.tsx @@ -0,0 +1,270 @@ +"use client"; + +import { useEffect, useState } from "react"; + +import { Legend, LineChart } from "@/components/charts"; +import { Button, Card, Chip, ErrorNote, Loading, Stat, Table, Td, cx, inputClass } from "@/components/ui"; +import { ApiError, api } from "@/lib/api"; +import { num, pct, taka } from "@/lib/format"; +import type { BusinessAssumptions, BusinessCase, BusinessPoint } from "@/lib/types"; + +/** Send-money and cash-out payments a month: orders of magnitude, not provider figures. */ +const VOLUMES = [ + { label: "upay scale", value: 20_000_000 }, + { label: "bKash scale", value: 300_000_000 }, +] as const; + +/** The assumptions a CFO is most likely to change, as the input shows them. */ +type Edited = Pick; + +function signed(value: number): string { + if (!value) return "same as today"; + return `${value > 0 ? "+" : "−"}${taka(Math.abs(value), true)} vs today`; +} + +function NumberField({ + label, value, onChange, min, max, step, suffix, +}: { + label: string; value: number; onChange: (value: number) => void; min: number; max: number; step: number; suffix?: string; +}) { + return ( + + ); +} + +/** Low and high net benefit of one assumption, drawn around today's net benefit. */ +function Tornado({ sensitivity, scale }: { sensitivity: BusinessCase["sensitivity"]; scale: [number, number] }) { + const [lo, hi] = scale; + const at = (value: number) => ((value - lo) / (hi - lo || 1)) * 100; + const base = at(sensitivity.net_benefit); + return ( + + {sensitivity.rows.map((row) => { + const left = Math.min(row.net_low, row.net_high); + const right = Math.max(row.net_low, row.net_high); + const fmt = (v: number) => (row.unit === "share" ? pct(v, 0) : num(v, 2)); + return ( + + + + + + + ); + })} +
{row.label}{fmt(row.low)} – {fmt(row.high)}{taka(row.net_low, true)} → {taka(row.net_high, true)} +
+
= row.net_low ? "bg-good/70" : "bg-bad/70")} + style={{ left: `${at(left)}%`, width: `${Math.max(at(right) - at(left), 0.5)}%` }} + /> +
+ {lo < 0 &&
} +
+
+ ); +} + +export function BusinessSection({ + index, setIndex, minutes, version, +}: { + index: number; setIndex: (index: number) => void; minutes: number; version: string; +}) { + const [edited, setEdited] = useState(null); + const [state, setState] = useState<{ data?: BusinessCase; error: ApiError | null }>({ error: null }); + + const body = edited ? { ...edited, review_minutes: minutes } : { review_minutes: minutes }; + const key = JSON.stringify(body); + useEffect(() => { + const controller = new AbortController(); + const timer = window.setTimeout(() => { + api(`/v1/model/business?version=${version}`, JSON.parse(key), controller.signal).then( + (data) => setState({ data, error: null }), + (error) => { + if (error?.name !== "AbortError") setState((old) => ({ data: old.data, error })); + }, + ); + }, 250); + return () => { + window.clearTimeout(timer); + controller.abort(); + }; + }, [key, version]); + + const data = state.data; + if (!data) return state.error ? : ; + + const a: Edited = edited ?? data.assumptions; + const change = (patch: Partial) => setEdited({ ...a, ...patch }); + const today = data.policy; + const point: BusinessPoint = index === data.today_index ? today : data.points[index]; + const best = data.points[data.best_index]; + const xs = data.points.map((p) => p.alert_rate * 100); + const labels = data.points.map((p) => pct(p.alert_rate, 2)); + const money = [ + { name: "Scam money kept from fraudsters", color: "var(--color-good)", values: data.points.map((p) => p.prevented_total) }, + { name: "Cost of running it (friction, analysts, platform)", color: "var(--color-bad)", values: data.points.map((p) => p.operating_cost) }, + ]; + const nets = data.sensitivity.rows.flatMap((r) => [r.net_low, r.net_high]); + const span: [number, number] = [Math.min(0, ...nets), Math.max(...nets)]; + const breakEven = data.break_even_bps; + + return ( + <> +
+
+

Business case, a month at your volume

+

+ The same threshold, priced in taka for {num(a.monthly_payments)} send-money and cash-out payments a month and a scam rate of{" "} + {num(a.scam_loss_bps, 2)} basis points of payment value ({taka(data.scam_loss_at_risk, true)} at risk a month). +

+
+ +
+ +
+ = 0 ? "good" : "bad"} + value={taka(point.net_benefit, true)} + sub={`${signed(point.net_benefit - today.net_benefit)} · to the provider alone ${taka(point.provider_net_benefit, true)}`} + /> + + today.friction_total ? "warn" : "plain"} + value={taka(point.friction_total, true)} + sub={`${num(point.abandoned_payments)} honest payments abandoned, ${num(point.support_contacts)} support contacts · ${num(point.honest_per_10k, 1)} honest customers interrupted per 10,000 payments`} + /> + +
+ +
+ + taka(v, true)} xTitle="Share of payments interrupted" mark={{ index, label: "selected" }} /> +
+ + +
+
+
+ {VOLUMES.map((v) => ( + change({ monthly_payments: v.value })}> + {v.label} + + ))} +
+
+ change({ monthly_payments: Math.round(v * 1e6) })} + min={0.01} + max={2000} + step={1} + /> +
+
+ change({ analyst_monthly_cost_taka: v })} + min={1000} + max={2_000_000} + step={5000} + suffix="BDT" + /> + change({ abandon_rate: v / 100 })} + min={0} + max={100} + step={1} + suffix="%" + /> + change({ scam_loss_bps: v })} + min={0.01} + max={100} + step={0.5} + suffix="basis points" + /> +

+ Analyst minutes per hold come from the assumption above. Running the policy in force pays for itself above{" "} + {breakEven.net_benefit === null ? "–" : `${num(breakEven.net_benefit, 2)} bp`} of scam losses + {breakEven.provider_net_benefit === null ? "" : ` (${num(breakEven.provider_net_benefit, 2)} bp counting the provider’s own money only)`}. +

+ +
+ +
+ +
+ + + +
+ +
+ Where each assumption comes from + + {data.sources.map((s) => ( + + + + + + ))} +
{s.label} + {s.unit === "share" ? pct(data.assumptions[s.key], 0) : num(data.assumptions[s.key], 2)} + {s.unit === "share" ? "" : ` ${s.unit}`} + {s.source}
+
+ +

+ Rates come from the back-test of model {data.model_version}. The simulator’s scams take{" "} + {taka(data.synthetic.scam_loss_per_payment)} per payment, far more than real traffic, so scam money and true alerts are scaled by{" "} + {num(data.synthetic.scale, 4)} to the assumed rate. These are estimates on synthetic data to show the shape of the + trade-off, not a forecast: a provider would replace the assumptions with its own figures first. See docs/BUSINESS_CASE.md. +

+ + ); +} diff --git a/frontend/app/(console)/impact/page.tsx b/frontend/app/(console)/impact/page.tsx index c69728a..a140e1d 100644 --- a/frontend/app/(console)/impact/page.tsx +++ b/frontend/app/(console)/impact/page.tsx @@ -8,6 +8,8 @@ import { useApi } from "@/lib/api"; import { num, pct, taka } from "@/lib/format"; import type { ImpactPoint, Insights, Report } from "@/lib/types"; +import { BusinessSection } from "./business"; + const TIER_NAME = { warn: "Warn", step_up: "Step-up check", hold: "Hold for review" } as const; /** Hours of case work one reviewer gets through in a shift. */ const REVIEW_HOURS = 6; @@ -154,6 +156,8 @@ function Simulator({ insights, version }: { insights: Insights; version: string + +

Measured by replaying model {version} over {num(insights.rows)} payments from {days} days it never trained on. The fraud is simulated, so the taka figures show the shape of the trade-off, not a forecast for a real customer base. Moving the slider changes nothing in production: thresholds @@ -169,7 +173,7 @@ export default function ImpactPage() { <> {(data) => diff --git a/frontend/app/(console)/model/page.tsx b/frontend/app/(console)/model/page.tsx index 5ac4c8c..1eabef2 100644 --- a/frontend/app/(console)/model/page.tsx +++ b/frontend/app/(console)/model/page.tsx @@ -6,7 +6,7 @@ import { HBars } from "@/components/charts"; import { Async, Badge, Card, Empty, Facts, PageHeader, Stat, StatusBadge, Table, Tabs, Td } from "@/components/ui"; import { useApi } from "@/lib/api"; import { day, num, pct, shortDay, TIER_LABEL, TIERS, when, words } from "@/lib/format"; -import type { Drift, Feedback, Insights, Registry, Report, Shadow, Summary } from "@/lib/types"; +import type { Drift, Feedback, Insights, LabelRealism, Registry, Report, Shadow, Spread, Summary } from "@/lib/types"; type Tab = "performance" | "drift" | "versions" | "feedback"; @@ -197,6 +197,7 @@ function Performance({ report, summary }: { report: Report; summary?: Summary }) /> + {report.label_realism && }

Model {report.model_version}, trained on {num(report.model.data.rows.train)} payments and measured on {num(report.model.data.rows.test)} later ones it never saw. The data is simulated: these numbers say the pipeline works end to end, not how it would do on real traffic. @@ -205,6 +206,48 @@ function Performance({ report, summary }: { report: Report; summary?: Summary }) ); } +const LABEL_REGIME: Record = { + ground_truth: "Every scam labelled (what the model was trained on)", + reported_uniform: "Only reported scams, half at random", + reported: "Only reported scams, as victims really report", + reported_pu: "+ positive-unlabelled learning", + reported_pu_known_rate: "+ positive-unlabelled, report rate known", + reported_propagated: "+ labels spread through mule wallets", + reported_feedback: "+ analyst verdicts on alerts", + reported_propagated_feedback: "+ mule-wallet labels and verdicts", +}; + +function spread(value: Spread | undefined): string { + return value ? `${pct(value.mean)} ± ${num(value.sd * 100, 1)}` : "–"; +} + +function UnderReporting({ data }: { data: LabelRealism }) { + const rows = data.regimes.filter((name) => data.summary.regimes[name]); + return ( +

+ + + {rows.map((name) => { + const row = data.summary.regimes[name]; + return ( + + + + + + + ); + })} +
{LABEL_REGIME[name] ?? words(name)}{spread(row.case_recall)}{spread(row.by_typology.investment_scam?.case_recall)}{row.pr_auc ? `${num(row.pr_auc.mean, 3)} ± ${num(row.pr_auc.sd, 3)}` : "–"}
+ +
+ ); +} + function DriftView({ drift, backtest }: { drift: Drift; backtest?: Insights["drift"] }) { const worst = [...drift.features].sort((a, b) => b.psi - a.psi).slice(0, 14); return ( diff --git a/frontend/app/(console)/phone/copy.ts b/frontend/app/(console)/phone/copy.ts new file mode 100644 index 0000000..8b24688 --- /dev/null +++ b/frontend/app/(console)/phone/copy.ts @@ -0,0 +1,196 @@ +// What the demo phone says, in the customer's language. The Bangla is written for +// the screen, the way upay's app speaks to its customers, not translated word by word. + +import type { AppealRelation, Cue, Tier } from "@/lib/types"; + +export type Lang = "bn" | "en"; +export type Words = Record; + +const BN_DIGITS = "০১২৩৪৫৬৭৮৯"; + +/** Bangla numerals for a Bangla screen; amounts, times and counts all read naturally. */ +export function digits(text: string | number, lang: Lang): string { + const value = String(text); + return lang === "bn" ? value.replace(/[0-9]/g, (d) => BN_DIGITS[Number(d)]) : value; +} + +/** mm:ss, or h:mm:ss past an hour. */ +export function clockText(seconds: number, lang: Lang): string { + const s = Math.max(0, Math.ceil(seconds)); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + const pad = (n: number) => String(n).padStart(2, "0"); + return digits(h ? `${h}:${pad(m)}:${pad(s % 60)}` : `${pad(m)}:${pad(s % 60)}`, lang); +} + +/** The spoken form for screen readers: "12 minutes 5 seconds". */ +export function clockSpoken(seconds: number, lang: Lang): string { + const s = Math.max(0, Math.ceil(seconds)); + const m = Math.floor(s / 60); + return lang === "bn" ? `${digits(m, lang)} মিনিট ${digits(s % 60, lang)} সেকেন্ড` : `${m} minutes ${s % 60} seconds`; +} + +/** The warning itself, by the kind of scam the decision looks like (the API's `cue`). */ +export const CUE: Record = { + reported_recipient: { + title: { bn: "এই নম্বরের বিরুদ্ধে প্রতারণার অভিযোগ আছে", en: "This number has been reported for fraud" }, + body: { + bn: "অন্য গ্রাহকেরা এই নম্বরে টাকা পাঠিয়ে প্রতারিত হয়েছেন। যাকে চেনেন না, বা কেউ তাড়া দিচ্ছে বলে পাঠাচ্ছেন — এমন হলে টাকা পাঠাবেন না।", + en: "Other customers lost money sending to this number. Don't send if you don't know them, or if someone is rushing you.", + }, + check: { bn: "প্রাপককে নিজে ফোন করে নিশ্চিত হোন।", en: "Call the person yourself to make sure it's them." }, + }, + impersonation: { + title: { bn: "কেউ কি upay, ব্যাংক বা সরকারি অফিসের লোক পরিচয় দিয়েছে?", en: "Did someone say they're from upay, a bank or the government?" }, + body: { + bn: "upay কখনো ফোন করে পিন, ওটিপি বা টাকা চায় না। কেউ চাইলে ফোন কেটে দিন।", + en: "upay never calls to ask for your PIN, OTP or money. If someone does, hang up.", + }, + check: { bn: "সন্দেহ হলে নিজে ১৬২৬৮ নম্বরে কল করুন।", en: "If in doubt, call 16268 yourself." }, + }, + prize: { + title: { bn: "পুরস্কার পেতে টাকা লাগে না", en: "Real prizes don't cost money" }, + body: { + bn: "লটারি, পুরস্কার বা উপহার পাওয়ার আগে ফি, ট্যাক্স বা চার্জ পাঠাতে বললে সেটা প্রতারণা।", + en: "If you're asked to pay a fee, tax or charge before you get a prize or gift, it's a scam.", + }, + check: { bn: "আপনি কি সত্যিই কোনো প্রতিযোগিতায় অংশ নিয়েছিলেন?", en: "Did you actually enter any contest?" }, + }, + investment: { + title: { bn: "নিশ্চিত লাভের প্রতিশ্রুতি? সাবধান", en: "Guaranteed profit? Be careful" }, + body: { + bn: "অল্প দিনে টাকা দ্বিগুণ বা নিশ্চিত মুনাফার কথা বলে টাকা নেওয়া একটি পরিচিত প্রতারণা।", + en: "Promises to double your money quickly, or of guaranteed profit, are a well-known scam.", + }, + check: { bn: "পাঠানোর আগে পরিবারের কারো সাথে কথা বলুন।", en: "Talk to someone in your family before you send." }, + }, + wrong_send: { + title: { bn: "কেউ কি 'ভুল করে পাঠিয়েছি' বলে টাকা ফেরত চাইছে?", en: "Is someone asking for money back they 'sent by mistake'?" }, + body: { + bn: "আগে নিজের ব্যালেন্স আর লেনদেনের তালিকা দেখুন। টাকা সত্যিই এসেছে কি না না দেখে কিছু ফেরত পাঠাবেন না।", + en: "Check your balance and your transaction list first. Don't send anything back until you've seen the money arrive.", + }, + check: { bn: "এসএমএস নয়, অ্যাপের লেনদেন তালিকা দেখুন — এসএমএস নকল হতে পারে।", en: "Check the app's history, not an SMS: an SMS can be faked." }, + }, + not_you: { + title: { bn: "লেনদেনটি কি আপনি নিজেই করছেন?", en: "Is this really you?" }, + body: { + bn: "এটি আপনার স্বাভাবিক লেনদেনের সাথে মিলছে না — নতুন ফোন, নতুন জায়গা, বা অস্বাভাবিক অঙ্ক।", + en: "This doesn't look like how you usually pay: a new phone, a new place or an unusual amount.", + }, + check: { bn: "আপনি না করে থাকলে বাতিল করে ১৬২৬৮ নম্বরে কল করুন।", en: "If it isn't you, cancel and call 16268." }, + }, + generic: { + title: { bn: "পাঠানোর আগে একটু ভাবুন", en: "Take a moment before you send" }, + body: { + bn: "এই লেনদেনটি পরিচিত প্রতারণার ধরনের সাথে মিলে যাচ্ছে।", + en: "This payment looks like a known scam pattern.", + }, + check: { bn: "প্রাপককে চেনেন তো? কেউ তাড়া দিচ্ছে না তো?", en: "Do you know the person? Is anyone rushing you?" }, + }, +}; + +/** The strip under the bar: what kind of interruption this is. */ +export const STRIP_TEXT: Record, Words> = { + warn: { bn: "একটু থামুন", en: "Pause a moment" }, + step_up: { bn: "আপনাকে আবার যাচাই করতে হবে", en: "Verify it's you" }, + hold: { bn: "লেনদেন সাময়িকভাবে স্থগিত", en: "Payment paused for review" }, +}; + +export const RELATIONS: [AppealRelation, Words][] = [ + ["family", { bn: "পরিবারের সদস্য", en: "Family" }], + ["friend", { bn: "বন্ধু", en: "Friend" }], + ["seller", { bn: "দোকান বা বিক্রেতা", en: "A shop or seller" }], + ["business", { bn: "ব্যবসার কাজে", en: "Business" }], + ["landlord", { bn: "বাড়িওয়ালা", en: "Landlord" }], + ["employer", { bn: "অফিস বা মালিক", en: "Employer" }], + ["other", { bn: "অন্য কেউ", en: "Someone else" }], + ["none", { bn: "চিনি না", en: "I don't know them" }], +]; + +/** Everything else on the phone, keyed by what it is for. */ +export const T = { + appName: { bn: "সেন্ড মানি", en: "Send money" }, + switchTo: { bn: "English", en: "বাংলা" }, + switchLabel: { bn: "Switch the app to English", en: "অ্যাপটি বাংলায় দেখুন" }, + back: { bn: "পেছনে যান", en: "Back" }, + to: { bn: "প্রাপক", en: "To" }, + amount: { bn: "পরিমাণ", en: "Amount" }, + balance: { bn: "বর্তমান ব্যালেন্স", en: "Balance" }, + from: { bn: "যে অ্যাকাউন্ট থেকে", en: "From" }, + chooseLeft: { bn: "শুরু করতে বাঁ দিক থেকে একটি লেনদেন বেছে নিন।", en: "Choose a payment on the left to begin." }, + helpline: { bn: "হেল্পলাইন ১৬২৬৮", en: "Helpline 16268" }, + beforeYouSend: { bn: "পাঠানোর আগে দেখুন", en: "Before you send" }, + send: { bn: "সেন্ড মানি করুন", en: "Send" }, + sending: { bn: "পাঠানো হচ্ছে…", en: "Sending…" }, + cancel: { bn: "বাতিল করুন", en: "Cancel the payment" }, + sendAnyway: { bn: "বুঝেছি, তবুও পাঠাব", en: "I understand, send anyway" }, + pin: { bn: "পিন দিন", en: "Enter your PIN" }, + verifySend: { bn: "যাচাই করে পাঠান", en: "Verify and send" }, + availableIn: { bn: "পাঠানো যাবে আর", en: "You can send in" }, + coolingOff: { + bn: "নিরাপত্তার জন্য এই অপেক্ষা। এই সময়ে ভেবে দেখুন কে আপনাকে টাকা পাঠাতে বলেছে। চাইলে এখনই বাতিল করতে পারেন।", + en: "This wait is for your safety. Use it to think about who asked you to pay. You can cancel at any time.", + }, + waitOver: { bn: "অপেক্ষার সময় শেষ। পিন দিয়ে পাঠাতে পারেন।", en: "The wait is over. Enter your PIN to send." }, + heldMoney: { bn: "আপনার টাকা আপনার অ্যাকাউন্টেই আছে।", en: "Your money is still in your account." }, + reviewIn: { bn: "একজন কর্মকর্তা দেখবেন, বাকি সময়", en: "A person will review it within" }, + reviewLate: { + bn: "পর্যালোচনায় নির্ধারিত সময়ের চেয়ে বেশি লাগছে। আমরা দুঃখিত — দরকার হলে ১৬২৬৮ নম্বরে কল করুন।", + en: "The review is taking longer than promised. We're sorry; call 16268 if you need to.", + }, + whatNext: { bn: "এরপর কী হবে", en: "What happens next" }, + report: { bn: "প্রতারণার অভিযোগ করুন", en: "Report a scam" }, + whatHappened: { bn: "কী হয়েছিল?", en: "What happened?" }, + reportDone: { bn: "অভিযোগ গ্রহণ করা হয়েছে।", en: "Report received." }, + appeal: { bn: "এটা আসল লেনদেন — আপিল করুন", en: "This payment is genuine: appeal" }, + appealTitle: { bn: "আপিল করুন", en: "Appeal" }, + appealIntro: { + bn: "আপনার মনে হলে লেনদেনটি সঠিক, আমাদের জানান। একজন কর্মকর্তা দেখে সিদ্ধান্ত নেবেন।", + en: "If you think this payment is fine, tell us. A person will look at it and decide.", + }, + relation: { bn: "প্রাপক আপনার কে হন?", en: "How do you know the person?" }, + reason: { bn: "কেন পাঠাচ্ছেন? (সংক্ষেপে)", en: "What is the payment for? (briefly)" }, + reasonHint: { bn: "পিন বা ওটিপি এখানে লিখবেন না।", en: "Never write your PIN or OTP here." }, + submitAppeal: { bn: "আপিল জমা দিন", en: "Send the appeal" }, + appealSent: { bn: "আপিল জমা হয়েছে", en: "Appeal received" }, + appealBy: { bn: "উত্তর পাবেন যত দেরিতে হলেও", en: "You'll get an answer by" }, + appealNoMoney: { + bn: "এই আপিলে অপেক্ষার সময় কমবে না; এটি আমাদের সতর্কবার্তা আরও ভালো করতে সাহায্য করবে।", + en: "An appeal doesn't shorten the wait; it helps us make our warnings better.", + }, + appealApproved: { bn: "আপিল গ্রহণ করা হয়েছে", en: "Your appeal was accepted" }, + appealRejected: { bn: "আপিল গ্রহণ করা হয়নি — কর্মকর্তা তদন্ত চালিয়ে যাচ্ছেন", en: "Your appeal wasn't accepted; the review continues" }, + checkStatus: { bn: "অবস্থা দেখুন", en: "Check the status" }, + sent: { bn: "টাকা পাঠানো হয়েছে", en: "Sent" }, + cancelled: { bn: "বাতিল হয়েছে", en: "Cancelled" }, + notSent: { bn: "লেনদেন হয়নি", en: "Not sent" }, + sentBody: { bn: "লেনদেন সফল হয়েছে।", en: "The payment went through." }, + cancelledBody: { bn: "কিছুই পাঠানো হয়নি। টাকা আপনার অ্যাকাউন্টেই আছে।", en: "Nothing was sent. Your money is still in your account." }, + refusedBody: { bn: "এই লেনদেনটি করা যায়নি। টাকা আপনার অ্যাকাউন্টেই আছে।", en: "This payment was refused. Your money is still in your account." }, + newPayment: { bn: "নতুন লেনদেন", en: "New payment" }, + minutes: { bn: "মিনিট", en: "minutes" }, +} satisfies Record; + +/** The steps after each kind of interruption: done, happening now, still to come. */ +export function steps(tier: Exclude, minutes: number | null, lang: Lang): { text: string; state: "done" | "now" | "next" }[] { + const n = digits(minutes ?? 30, lang); + const words = { + warn: [ + { bn: "আমরা আপনাকে সতর্ক করেছি", en: "We've warned you" }, + { bn: "আপনি ঠিক করুন: বাতিল, নাকি তবুও পাঠাবেন", en: "You decide: cancel, or send anyway" }, + { bn: "পাঠালে টাকা সাথে সাথে চলে যাবে, আর ফেরত আনা কঠিন", en: "If you send, the money goes at once and is hard to get back" }, + ], + step_up: [ + { bn: "আমরা লেনদেনটি থামিয়েছি", en: "We've paused the payment" }, + { bn: `${n} মিনিট অপেক্ষা — এর মধ্যে যেকোনো সময় বাতিল করা যায়`, en: `A ${n}-minute wait; you can cancel at any time` }, + { bn: "এরপর পিন দিয়ে পাঠাতে পারবেন", en: "Then you can send it with your PIN" }, + ], + hold: [ + { bn: "টাকা আপনার অ্যাকাউন্টেই আছে, কোথাও যায়নি", en: "Your money hasn't left your account" }, + { bn: `একজন কর্মকর্তা ${n} মিনিটের মধ্যে দেখবেন`, en: `A person reviews it within ${n} minutes` }, + { bn: "ঠিক থাকলে টাকা পৌঁছে যাবে; না হলে আপনার কাছেই থাকবে", en: "If it's fine, it's delivered; if not, it stays with you" }, + ], + }[tier]; + return words.map((step, index) => ({ text: step[lang], state: index === 0 ? "done" : index === 1 ? "now" : "next" })); +} diff --git a/frontend/app/(console)/phone/page.tsx b/frontend/app/(console)/phone/page.tsx index b35f1f6..d6b86b4 100644 --- a/frontend/app/(console)/phone/page.tsx +++ b/frontend/app/(console)/phone/page.tsx @@ -2,15 +2,16 @@ import Image from "next/image"; import Link from "next/link"; -import { type ReactNode, useEffect, useState } from "react"; +import { type ReactNode, useEffect, useId, useRef, useState } from "react"; import { ScoreBar } from "@/components/charts"; import { useSession } from "@/components/session"; import { Async, Badge, Button, Card, cx, ErrorNote, Facts, inputClass, PageHeader, StatusBadge, TierBadge } from "@/components/ui"; import { api, ApiError, qs, useApi } from "@/lib/api"; -import { maskId, num, taka, when, words } from "@/lib/format"; -import type { PayResult, Scenario, Text2, Tier } from "@/lib/types"; +import { maskId, taka, when, words } from "@/lib/format"; +import type { AppealRelation, CustomerAppeal, PayResult, Scenario, Text2, Tier } from "@/lib/types"; +import { clockSpoken, clockText, CUE, digits, type Lang, RELATIONS, steps, STRIP_TEXT, T, type Words } from "./copy"; // upay's app icon. The name and the mark belong to UCB Fintech Company Limited. import upayLogo from "./upay-logo.png"; @@ -42,9 +43,9 @@ const NETWORKS = [ const SCENARIO: Record = { allow: { title: "An ordinary payment", text: "Goes straight through; the customer never sees FraudLens." }, - warn: { title: "Looks like a scam", text: "The customer gets a Bangla warning and chooses." }, + warn: { title: "Looks like a scam", text: "The customer gets a warning naming the scam, and chooses." }, step_up: { title: "Riskier", text: "Verify again, then wait out a cooling-off period." }, - hold: { title: "Very likely fraud", text: "Paused until a reviewer decides." }, + hold: { title: "Very likely fraud", text: "Paused until a reviewer decides; the customer can appeal." }, confirmed_fraud: { title: "To a confirmed-fraud wallet", text: "Held by a hard rule, whatever the model says." }, }; const CATEGORIES: [string, string, string][] = [ @@ -60,27 +61,39 @@ const CATEGORIES: [string, string, string][] = [ ["other", "অন্য কিছু", "Something else"], ]; const WALLET_ID = /^[A-Za-z0-9_-]{1,32}$/; +const LANG_KEY = "fraudlens.phone.lang"; +const REASON_MAX = 500; // The phone is drawn the way upay's app looks: its yellow bar, blue buttons and white pages, // whatever the console around it looks like. What FraudLens adds is the strip under the bar. +// Every pairing here passes WCAG AA contrast for its text size. const STRIP: Record = { allow: "border-green-300 bg-green-50 text-green-900", - warn: "border-amber-300 bg-amber-50 text-amber-900", - step_up: "border-orange-300 bg-orange-50 text-orange-900", + warn: "border-amber-300 bg-amber-50 text-amber-950", + step_up: "border-orange-300 bg-orange-50 text-orange-950", hold: "border-red-300 bg-red-50 text-red-900", }; // The services on upay's home screen. Only send money does anything here. const SERVICES: [string, string, string][] = [ ["সেন্ড মানি", "Send money", "M3 11.5 21 3l-6.5 18-3-7.5z"], - ["মোবাইল রিচার্জ", "Mobile recharge", "M8 2.5h8A1.5 1.5 0 0 1 17.5 4v16a1.5 1.5 0 0 1-1.5 1.5H8A1.5 1.5 0 0 1 6.5 20V4A1.5 1.5 0 0 1 8 2.5zM10.5 18.5h3"], + ["মোবাইল রিচার্জ", "Recharge", "M8 2.5h8A1.5 1.5 0 0 1 17.5 4v16a1.5 1.5 0 0 1-1.5 1.5H8A1.5 1.5 0 0 1 6.5 20V4A1.5 1.5 0 0 1 8 2.5zM10.5 18.5h3"], ["ক্যাশ আউট", "Cash out", "M12 3v12m0 0-4-4m4 4 4-4M4 20h16"], - ["মেক পেমেন্ট", "Make payment", "M4 4h6v6H4zM14 4h6v6h-6zM4 14h6v6H4zM14 14h2v2h-2zM18 18h2v2h-2zM18 14h2M14 18v2"], + ["মেক পেমেন্ট", "Payment", "M4 4h6v6H4zM14 4h6v6h-6zM4 14h6v6H4zM14 14h2v2h-2zM18 18h2v2h-2zM18 14h2M14 18v2"], ["অ্যাড মানি", "Add money", "M12 3a9 9 0 1 0 0 18 9 9 0 0 0 0-18zM12 8v8M8 12h8"], ["পে বিল", "Pay bill", "M6 3h12v18l-3-2-3 2-3-2-3 2zM9 8h6M9 12h6"], - ["রিকোয়েস্ট মানি", "Request money", "M12 21V9m0 0-4 4m4-4 4 4M4 4h16"], - ["ফান্ড ট্রান্সফার", "Fund transfer", "M3 9.5 12 4l9 5.5M5 10v8M9.5 10v8M14.5 10v8M19 10v8M3 20h18"], + ["রিকোয়েস্ট মানি", "Request", "M12 21V9m0 0-4 4m4-4 4 4M4 4h16"], + ["ফান্ড ট্রান্সফার", "Transfer", "M3 9.5 12 4l9 5.5M5 10v8M9.5 10v8M14.5 10v8M19 10v8M3 20h18"], ]; +/** The language the demo customer last chose. Storage can be missing or blocked: Bangla then. */ +function savedLang(): Lang { + try { + return window.localStorage.getItem(LANG_KEY) === "en" ? "en" : "bn"; + } catch { + return "bn"; + } +} + function Icon({ d, className }: { d: string; className?: string }) { return ( {T.switchTo[lang]} + + ); +} + // The buttons stay at the bottom of the screen while a long message scrolls behind them. const FOOT = "sticky -bottom-4 -mx-4 -mb-4 mt-auto space-y-2 border-t border-neutral-200 bg-white p-4"; +/** One screen of the app. Its title takes focus when it appears, so a screen reader + * announces the new screen and keyboard users start from the top of it. */ function Screen({ - title, tone = "plain", heading, onBack, children, -}: { title: ReactNode; tone?: Tier | "plain"; heading?: ReactNode; onBack?: () => void; children: ReactNode }) { + title, tone = "plain", heading, onBack, lang, onLang, children, +}: { + title: ReactNode; tone?: Tier | "plain"; heading?: ReactNode; onBack?: () => void; + lang: Lang; onLang: () => void; children: ReactNode; +}) { + const titleRef = useRef(null); + useEffect(() => titleRef.current?.focus(), []); return (
-
+
{onBack && ( - )} - {title} +

{title}

+
{tone !== "plain" && heading && (
{heading}
@@ -128,43 +164,31 @@ function Screen({ ); } -function Home() { +function Home({ lang, onLang }: { lang: Lang; onLang: () => void }) { return (
upay
-
Demo customer
-
01XXXXXXXXX
+
{lang === "bn" ? "ডেমো গ্রাহক" : "Demo customer"}
+
01XXXXXXXXX
- ব্যালেন্স +
    {SERVICES.map(([bn, en, d], index) => ( -
  • 0 && "opacity-60")}> +
  • 0 && "opacity-60")}> - {bn} + {lang === "bn" ? bn : en}
    • ))}
-
সেন্ড মানি
- Choose a payment on the left to begin. +
{T.appName[lang]}
+ {T.chooseLeft[lang]}
-

- হেল্পলাইন ১৬২৬৮ · Helpline 16268 -

-
- ); -} - -function Message({ text }: { text: Text2 | null | undefined }) { - if (!text) return null; - return ( -
-

{text.bn}

-

{text.en}

+

{T.helpline[lang]}

); } @@ -173,13 +197,82 @@ function PhoneButton({ tone = "dark", ...rest }: React.ButtonHTMLAttributes; + return ( + + )} +
); - const label = "text-xs font-semibold text-neutral-500"; + const label = "text-xs font-semibold text-neutral-600"; const avatar = ( ); + const amount = payment ? digits(taka(payment.amount), lang) : ""; // One row once there is a decision to read, so the message and the buttons get the room. const party = payment && (result ? (
{avatar} - প্রাপক · To + {w(T.to)} {maskId(payment.receiver_id)} - {taka(payment.amount)} + {amount}
) : (
-
প্রাপক · To
+
{w(T.to)}
{avatar} {maskId(payment.receiver_id)}
-
অ্যামাউন্ট · Amount
-
{taka(payment.amount)}
+
{w(T.amount)}
+
{amount}
)); - const sendMoney = <>সেন্ড মানি · Send money; + const sendMoney = w(T.appName); // Back to the home screen. Not offered while a warning waits for the customer's answer. - const home = busy ? undefined : () => { setPayment(null); setPicked(null); setResult(null); setOutcome(null); setReported(null); setError(null); }; + const home = busy ? undefined : () => { setPayment(null); setPicked(null); setResult(null); setOutcome(null); setReported(null); setAppeal(null); setAppealing(false); setError(null); setWait(null); setReview(null); }; + const screenProps = { lang, onLang: switchLang }; let screen: ReactNode; + let screenKey: string; if (!payment) { - screen = ; + screenKey = "home"; + screen = ; } else if (reporting) { + screenKey = "report"; screen = ( - কী হয়েছিল? · What happened?} onBack={() => setReporting(false)}> + setReporting(false)} {...screenProps}> {CATEGORIES.map(([id, bn, en]) => ( ))} ); + } else if (appealing && decision) { + screenKey = "appeal"; + const left = REASON_MAX - reason.length; + screen = ( + setAppealing(false)} {...screenProps}> +
+

{w(T.appealIntro)}

+
+ {w(T.relation)} +
+ {RELATIONS.map(([value, text]) => ( + + ))} +
+
+