diff --git a/.gitignore b/.gitignore index 8487cfeb..341332e8 100644 --- a/.gitignore +++ b/.gitignore @@ -162,11 +162,12 @@ cython_debug/ .idea/ certs/ -core/static/jettons/* -core/static/nfts/* -core/static/chats/* -core/static/avatars/* *.session frontend/public/settings.json + + +# AI +.agents/ +.antigravitycli/ \ No newline at end of file diff --git a/.specify/.gitignore b/.specify/.gitignore new file mode 100644 index 00000000..e314fc2f --- /dev/null +++ b/.specify/.gitignore @@ -0,0 +1,9 @@ +# Machine-local Spec Kit state — not meant to be shared. +# Managed by the Specify CLI; safe to edit (your changes are preserved on refresh). + +# Local pointer to the current feature directory. Rewritten every time you +# switch features, so it is per-checkout state rather than something to share. +feature.json + +# Per-machine extension config overrides. +extensions/*/local-config.yml diff --git a/.specify/init-options.json b/.specify/init-options.json new file mode 100644 index 00000000..c8553faa --- /dev/null +++ b/.specify/init-options.json @@ -0,0 +1,9 @@ +{ + "ai": "agy", + "ai_skills": true, + "feature_numbering": "sequential", + "here": false, + "integration": "agy", + "script": "sh", + "speckit_version": "1.0.11" +} diff --git a/.specify/integration.json b/.specify/integration.json new file mode 100644 index 00000000..2899e309 --- /dev/null +++ b/.specify/integration.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.11", + "integration_state_schema": 1, + "installed_integrations": [ + "agy" + ], + "integration_settings": { + "agy": { + "script": "sh", + "invoke_separator": "-" + } + }, + "integration": "agy", + "default_integration": "agy" +} diff --git a/.specify/integrations/agy.manifest.json b/.specify/integrations/agy.manifest.json new file mode 100644 index 00000000..d3247b3f --- /dev/null +++ b/.specify/integrations/agy.manifest.json @@ -0,0 +1,17 @@ +{ + "integration": "agy", + "version": "1.0.11", + "installed_at": "2026-09-24T18:54:51.211894+00:00", + "files": { + ".agents/skills/speckit-analyze/SKILL.md": "60cbd0a4bb5c01bc5d447e5d42543b02ebf1560a2b9bfcb750b3502c5941a711", + ".agents/skills/speckit-clarify/SKILL.md": "457e6985a30cf6b3d89990b9ab0f198803398c5cd1a35b62f4ee3e59c8dc30d3", + ".agents/skills/speckit-constitution/SKILL.md": "177600a10d466eac78014a3b65d78663e24c05463266df539b5be9830a81c18e", + ".agents/skills/speckit-implement/SKILL.md": "0c565295d8b1de03821812be86e6d5bba68268f40d52550499110ab9cad1b118", + ".agents/skills/speckit-converge/SKILL.md": "90c6a2cf400d114a41945745aeadc94ebcd69f4eb12dd723635ca153806bced6", + ".agents/skills/speckit-plan/SKILL.md": "c77d361665b34d5870dd1ffe9d2c805700a36cfacd35c63a935ea6edcb100762", + ".agents/skills/speckit-checklist/SKILL.md": "43009036898a3242742336fd3582c2c5909d12fd9a7d6894eb8608b43ccd27ad", + ".agents/skills/speckit-specify/SKILL.md": "342dd2173103ce701b2bae2f489b462da19fc7369d920c93df10754be2216be8", + ".agents/skills/speckit-tasks/SKILL.md": "4dfb5cb2fd544f7ad5f1ddb9ff44bf230082fc312090db7e36102fe400dfab77", + ".agents/skills/speckit-taskstoissues/SKILL.md": "a25c75b567310292548df39ebaef19b98e3fa9c0b28de32a177c677256acf743" + } +} diff --git a/.specify/integrations/speckit.manifest.json b/.specify/integrations/speckit.manifest.json new file mode 100644 index 00000000..00fa03e4 --- /dev/null +++ b/.specify/integrations/speckit.manifest.json @@ -0,0 +1,19 @@ +{ + "integration": "speckit", + "version": "1.0.11", + "installed_at": "2026-09-24T18:54:51.226166+00:00", + "files": { + ".specify/scripts/bash/check-prerequisites.sh": "daa377146db4fb69912f42611a7b91c5d55873b3e3e27ed33bd8d67505826344", + ".specify/scripts/bash/create-new-feature.sh": "7f85da63527be6464b08e79578b35b33d0b259fa7653a61ae64da626a26207e2", + ".specify/scripts/bash/resolve-template.sh": "829e227096abc8bf0889889ec9f792f503ca5d395b7836a8a7eb739ee75214e7", + ".specify/scripts/bash/setup-plan.sh": "d0bd298c026446f3c63167130d65e49e4aa4485de6bef18bcdfd57b19aa0de96", + ".specify/scripts/bash/common.sh": "adaec9a43ecaa52609eeb9661d5f9c36a41d3160aa2bd714be9973082d0dc2d2", + ".specify/scripts/bash/setup-tasks.sh": "4a33dd1e6c32ddc7d570f4b538572c54069192573eed0ec654d9fb826e31a4fe", + ".specify/templates/spec-template.md": "3945437fc35cd30a5b2bf7beea680337c3516826d3efa5a6b92c4a7eca1ba28e", + ".specify/templates/tasks-template.md": "fc29a233f6f5a27ca31f1aa46b596af6500c627441c6e62b2bc4a1d721525842", + ".specify/templates/constitution-template.md": "ce7549540fa45543cca797a150201d868e64495fdff39dc38246fb17bd4024b3", + ".specify/templates/checklist-template.md": "856532b3cb66171c662cc16f16b31a5856e4655a8666aad1e545bbfc7f603ca1", + ".specify/templates/plan-template.md": "7e637502d41eccf0ca672496636365691fdca62ef37b27ec07fcb412dbfa90d4", + ".specify/.gitignore": "8c908410d177a1ef3d0dee16d7ad55f2ac3333df3104c4d4adee1c9b82f1dbc1" + } +} diff --git a/.specify/memory/.constitution-template.json b/.specify/memory/.constitution-template.json new file mode 100644 index 00000000..2fe4dee3 --- /dev/null +++ b/.specify/memory/.constitution-template.json @@ -0,0 +1,4 @@ +{ + "sha256": "ce7549540fa45543cca797a150201d868e64495fdff39dc38246fb17bd4024b3", + "source": "core" +} diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md new file mode 100644 index 00000000..a4670ff4 --- /dev/null +++ b/.specify/memory/constitution.md @@ -0,0 +1,50 @@ +# [PROJECT_NAME] Constitution + + +## Core Principles + +### [PRINCIPLE_1_NAME] + +[PRINCIPLE_1_DESCRIPTION] + + +### [PRINCIPLE_2_NAME] + +[PRINCIPLE_2_DESCRIPTION] + + +### [PRINCIPLE_3_NAME] + +[PRINCIPLE_3_DESCRIPTION] + + +### [PRINCIPLE_4_NAME] + +[PRINCIPLE_4_DESCRIPTION] + + +### [PRINCIPLE_5_NAME] + +[PRINCIPLE_5_DESCRIPTION] + + +## [SECTION_2_NAME] + + +[SECTION_2_CONTENT] + + +## [SECTION_3_NAME] + + +[SECTION_3_CONTENT] + + +## Governance + + +[GOVERNANCE_RULES] + + +**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE] + diff --git a/.specify/scripts/bash/check-prerequisites.sh b/.specify/scripts/bash/check-prerequisites.sh new file mode 100755 index 00000000..a6a8ad81 --- /dev/null +++ b/.specify/scripts/bash/check-prerequisites.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash + +# Consolidated prerequisite checking script +# +# This script provides unified prerequisite checking for Spec-Driven Development workflow. +# It replaces the functionality previously spread across multiple scripts. +# +# Usage: ./check-prerequisites.sh [OPTIONS] +# +# OPTIONS: +# --json Output in JSON format +# --require-spec Require spec.md to exist (for analysis phase) +# --require-tasks Require tasks.md to exist (for implementation phase) +# --include-tasks Include tasks.md in AVAILABLE_DOCS list +# --paths-only Only output path variables (no validation) +# --template NAME Include composed template content in JSON output +# --help, -h Show help message +# +# OUTPUTS: +# JSON mode: {"FEATURE_DIR":"...", "AVAILABLE_DOCS":["..."]} +# Text mode: FEATURE_DIR:... \n AVAILABLE_DOCS: \n ✓/✗ file.md +# Paths only: REPO_ROOT: ... \n BRANCH: ... \n FEATURE_DIR: ... etc. + +set -e + +# Parse command line arguments +JSON_MODE=false +REQUIRE_SPEC=false +REQUIRE_TASKS=false +INCLUDE_TASKS=false +PATHS_ONLY=false +TEMPLATE_NAME="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --json) + JSON_MODE=true + ;; + --require-spec) + REQUIRE_SPEC=true + ;; + --require-tasks) + REQUIRE_TASKS=true + ;; + --include-tasks) + INCLUDE_TASKS=true + ;; + --paths-only) + PATHS_ONLY=true + ;; + --template) + shift + if [[ $# -eq 0 ]]; then + echo "ERROR: --template requires a template name" >&2 + exit 1 + fi + TEMPLATE_NAME="$1" + ;; + --help|-h) + cat << 'EOF' +Usage: check-prerequisites.sh [OPTIONS] + +Consolidated prerequisite checking for Spec-Driven Development workflow. + +OPTIONS: + --json Output in JSON format + --require-spec Require spec.md to exist (for analysis phase) + --require-tasks Require tasks.md to exist (for implementation phase) + --include-tasks Include tasks.md in AVAILABLE_DOCS list + --paths-only Only output path variables (no prerequisite validation) + --template NAME Include composed template content in JSON output + --help, -h Show this help message + +EXAMPLES: + # Check task prerequisites (plan.md required) + ./check-prerequisites.sh --json + + # Check implementation prerequisites (plan.md + tasks.md required) + ./check-prerequisites.sh --json --require-tasks --include-tasks + + # Get feature paths only (no validation) + ./check-prerequisites.sh --paths-only + +EOF + exit 0 + ;; + *) + echo "ERROR: Unknown option '$1'. Use --help for usage information." >&2 + exit 1 + ;; + esac + shift +done + +# Source common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get feature paths. +# In --paths-only mode this is pure resolution, so pass --no-persist to opt out +# of the feature.json write side effect (issue #3025). +if $PATHS_ONLY; then + _paths_output=$(get_feature_paths --no-persist) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +else + _paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +fi +eval "$_paths_output" +unset _paths_output + +# If paths-only mode, output paths and exit (no validation) +if $PATHS_ONLY; then + if $JSON_MODE; then + # Minimal JSON paths payload (no validation performed) + if has_jq; then + jq -cn \ + --arg repo_root "$REPO_ROOT" \ + --arg branch "$CURRENT_BRANCH" \ + --arg feature_dir "$FEATURE_DIR" \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg tasks "$TASKS" \ + '{REPO_ROOT:$repo_root,BRANCH:$branch,FEATURE_DIR:$feature_dir,FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,TASKS:$tasks}' + else + printf '{"REPO_ROOT":"%s","BRANCH":"%s","FEATURE_DIR":"%s","FEATURE_SPEC":"%s","IMPL_PLAN":"%s","TASKS":"%s"}\n' \ + "$(json_escape "$REPO_ROOT")" "$(json_escape "$CURRENT_BRANCH")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$TASKS")" + fi + else + echo "REPO_ROOT: $REPO_ROOT" + echo "BRANCH: $CURRENT_BRANCH" + echo "FEATURE_DIR: $FEATURE_DIR" + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "TASKS: $TASKS" + fi + exit 0 +fi + +# Validate required directories and files +if [[ ! -d "$FEATURE_DIR" ]]; then + echo "ERROR: Feature directory not found: $FEATURE_DIR" >&2 + echo "Run /speckit-specify first to create the feature structure." >&2 + exit 1 +fi + +if [[ ! -f "$IMPL_PLAN" ]]; then + echo "ERROR: plan.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit-plan first to create the implementation plan." >&2 + exit 1 +fi + +# Check for spec.md if required +if $REQUIRE_SPEC && [[ ! -f "$FEATURE_SPEC" ]]; then + echo "ERROR: spec.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit-specify first to create the feature specification." >&2 + exit 1 +fi + +# Check for tasks.md if required +if $REQUIRE_TASKS && [[ ! -f "$TASKS" ]]; then + echo "ERROR: tasks.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit-tasks first to create the task list." >&2 + exit 1 +fi + +# Build list of available documents +docs=() + +# Always check these optional docs +[[ -f "$RESEARCH" ]] && docs+=("research.md") +[[ -f "$DATA_MODEL" ]] && docs+=("data-model.md") + +# Check contracts directory (only if it exists and has files) +if [[ -d "$CONTRACTS_DIR" ]] && [[ -n "$(ls -A "$CONTRACTS_DIR" 2>/dev/null)" ]]; then + docs+=("contracts/") +fi + +[[ -f "$QUICKSTART" ]] && docs+=("quickstart.md") + +# Include tasks.md if requested and it exists +if $INCLUDE_TASKS && [[ -f "$TASKS" ]]; then + docs+=("tasks.md") +fi + +TEMPLATE_CONTENT="" +if [[ -n "$TEMPLATE_NAME" ]]; then + if TEMPLATE_CONTENT=$(resolve_template_content "$TEMPLATE_NAME" "$REPO_ROOT"; status=$?; printf x; exit "$status"); then + TEMPLATE_CONTENT="${TEMPLATE_CONTENT%x}" + else + echo "ERROR: Could not resolve required $TEMPLATE_NAME from the template override stack for $REPO_ROOT" >&2 + exit 1 + fi +fi + +# Output results +if $JSON_MODE; then + # Build JSON array of documents + if has_jq; then + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(printf '%s\n' "${docs[@]}" | jq -R . | jq -s .) + fi + if [[ -n "$TEMPLATE_NAME" ]]; then + jq -cn \ + --arg feature_dir "$FEATURE_DIR" \ + --argjson docs "$json_docs" \ + --arg template_content "$TEMPLATE_CONTENT" \ + '{FEATURE_DIR:$feature_dir,AVAILABLE_DOCS:$docs,TEMPLATE_CONTENT:$template_content}' + else + jq -cn \ + --arg feature_dir "$FEATURE_DIR" \ + --argjson docs "$json_docs" \ + '{FEATURE_DIR:$feature_dir,AVAILABLE_DOCS:$docs}' + fi + else + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(for d in "${docs[@]}"; do printf '"%s",' "$(json_escape "$d")"; done) + json_docs="[${json_docs%,}]" + fi + if [[ -n "$TEMPLATE_NAME" ]]; then + printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s,"TEMPLATE_CONTENT":"%s"}\n' \ + "$(json_escape "$FEATURE_DIR")" "$json_docs" "$(json_escape "$TEMPLATE_CONTENT")" + else + printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s}\n' "$(json_escape "$FEATURE_DIR")" "$json_docs" + fi + fi +else + # Text output + echo "FEATURE_DIR:$FEATURE_DIR" + echo "AVAILABLE_DOCS:" + + # Show status of each potential document + check_file "$RESEARCH" "research.md" + check_file "$DATA_MODEL" "data-model.md" + check_dir "$CONTRACTS_DIR" "contracts/" + check_file "$QUICKSTART" "quickstart.md" + + if $INCLUDE_TASKS; then + check_file "$TASKS" "tasks.md" + fi +fi diff --git a/.specify/scripts/bash/common.sh b/.specify/scripts/bash/common.sh new file mode 100755 index 00000000..59e68099 --- /dev/null +++ b/.specify/scripts/bash/common.sh @@ -0,0 +1,933 @@ +#!/usr/bin/env bash +# Common functions and variables for all scripts + +# Find repository root by searching upward for .specify directory +# This is the primary marker for spec-kit projects +find_specify_root() { + local dir="${1:-$(pwd)}" + # Normalize to absolute path to prevent infinite loop with relative paths + # Use -- to handle paths starting with - (e.g., -P, -L) + dir="$(cd -- "$dir" 2>/dev/null && pwd)" || return 1 + local prev_dir="" + while true; do + if [ -d "$dir/.specify" ]; then + echo "$dir" + return 0 + fi + # Stop if we've reached filesystem root or dirname stops changing + if [ "$dir" = "/" ] || [ "$dir" = "$prev_dir" ]; then + break + fi + prev_dir="$dir" + dir="$(dirname "$dir")" + done + return 1 +} + +# Resolve an explicit SPECIFY_INIT_DIR project override (the directory that +# *contains* .specify/), for non-interactive / CI use — e.g. running a Spec Kit +# command against a member project from a monorepo root without cd. +# +# Precondition: SPECIFY_INIT_DIR is non-empty. Echoes the validated absolute +# project root, or prints an error and returns 1. Strict by design: the path +# must exist and contain .specify/, with no silent fallback to cwd or the +# script-location default (which would silently write to the wrong project). +# +# This is the single resolver: bundled extensions inherit it by sourcing core +# (e.g. the git extension's create-new-feature-branch) rather than duplicating it. +resolve_specify_init_dir() { + local init_root + # Normalize: relative paths resolve against $(pwd); a trailing slash collapses. + # CDPATH="" so a relative value cannot be resolved against the caller's CDPATH + # (which would also echo to stdout and corrupt the captured path). + if ! init_root="$(CDPATH="" cd -- "$SPECIFY_INIT_DIR" 2>/dev/null && pwd)"; then + echo "ERROR: SPECIFY_INIT_DIR does not point to an existing directory: $SPECIFY_INIT_DIR" >&2 + return 1 + fi + if [[ ! -d "$init_root/.specify" ]]; then + echo "ERROR: SPECIFY_INIT_DIR is not a Spec Kit project (no .specify/ directory): $init_root" >&2 + return 1 + fi + printf '%s\n' "$init_root" +} + +# Get repository root, prioritizing .specify directory +# This prevents using a parent repository when spec-kit is initialized in a subdirectory +get_repo_root() { + # Explicit project override wins (see resolve_specify_init_dir). + if [[ -n "${SPECIFY_INIT_DIR:-}" ]]; then + resolve_specify_init_dir + return + fi + + # First, look for .specify directory (spec-kit's own marker) + local specify_root + if specify_root=$(find_specify_root); then + echo "$specify_root" + return + fi + + # Final fallback to script location + local script_dir="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + (cd "$script_dir/../../.." && pwd) +} + +# Get current feature name from explicit state only. +# Returns the feature identifier or empty string if none is set. +# Feature state is set by SPECIFY_FEATURE (from create-new-feature or +# the git extension) or implicitly via .specify/feature.json. +get_current_branch() { + if [[ -n "${SPECIFY_FEATURE:-}" ]]; then + echo "$SPECIFY_FEATURE" + return + fi + + # No explicit feature set — caller must handle this via feature.json + # in get_feature_paths(). Return empty to signal "unknown". + echo "" +} + +# Safely read .specify/feature.json's "feature_directory" value. +# Prints the raw value (possibly relative) to stdout, or empty string if the file +# is missing, unparseable, or does not contain the key. Always returns 0 so callers +# under `set -e` cannot be aborted by parser failure. +# Parser order mirrors the historical get_feature_paths behavior: jq -> python3 -> grep/sed. +read_feature_json_feature_directory() { + local repo_root="$1" + local fj="$repo_root/.specify/feature.json" + [[ -f "$fj" ]] || { printf '%s' ''; return 0; } + + # Try parsers in order (jq -> python3 -> grep/sed), falling through on + # failure. Selection is by *parse success*, not mere availability: on + # Windows `python3` commonly resolves to the Microsoft Store App Execution + # Alias stub, which passes `command -v` but fails at runtime (exit 49), so + # an availability-gated `elif` would pick python3, swallow its failure, and + # never reach the grep/sed fallback -- leaving feature.json unreadable even + # though it is valid (issue #3304). + local _fd='' + if command -v jq >/dev/null 2>&1; then + if ! _fd=$(jq -r '.feature_directory // empty' "$fj" 2>/dev/null); then + _fd='' + fi + fi + if [[ -z "$_fd" ]] && command -v python3 >/dev/null 2>&1; then + # Use Python so pretty-printed/multi-line JSON still parses correctly. + if ! _fd=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); v=d.get('feature_directory'); print(v if v else '')" "$fj" 2>/dev/null); then + _fd='' + fi + fi + if [[ -z "$_fd" ]]; then + # Last-resort single-line grep/sed fallback. The `|| true` guards against + # grep returning 1 (no match) aborting under `set -e` / `pipefail`. + _fd=$( { grep -E '"feature_directory"[[:space:]]*:' "$fj" 2>/dev/null || true; } \ + | head -n 1 \ + | sed -E 's/^[^:]*:[[:space:]]*"([^"]*)".*$/\1/' ) + fi + + printf '%s' "$_fd" + return 0 +} + +# Persist a feature_directory value to .specify/feature.json. +# Writes only when the file is missing or the value differs from what's stored. +# Accepts the raw (possibly relative) path — callers should pass the original +# user-supplied value, not the normalized absolute path. +_persist_feature_json() { + local repo_root="$1" + local feature_dir_value="$2" + local fj="$repo_root/.specify/feature.json" + + # Strip repo_root prefix if the value is absolute and under repo_root + if [[ "$feature_dir_value" == "$repo_root/"* ]]; then + feature_dir_value="${feature_dir_value#"$repo_root/"}" + fi + + # Read current value (if any) and skip write when unchanged + local current_val + current_val=$(read_feature_json_feature_directory "$repo_root") + if [[ "$current_val" == "$feature_dir_value" ]]; then + return 0 + fi + + # Ensure .specify/ directory exists + mkdir -p "$repo_root/.specify" + + # Write feature.json — prefer jq for safe JSON, fall back to printf + if command -v jq >/dev/null 2>&1; then + jq -cn --arg fd "$feature_dir_value" '{feature_directory:$fd}' > "$fj" + else + printf '{"feature_directory":"%s"}\n' "$(json_escape "$feature_dir_value")" > "$fj" + fi +} + +get_feature_paths() { + # Read-only callers (e.g. check-prerequisites.sh --paths-only) pass + # --no-persist so pure path resolution never writes .specify/feature.json, + # which would dirty the working tree or overwrite a pinned value (issue #3025). + local no_persist=false + if [[ "${1:-}" == "--no-persist" ]]; then + no_persist=true + shift + fi + # SPECIFY_FEATURE_NO_PERSIST is the environment-level equivalent of --no-persist, + # letting an orchestrator (multi-agent runner, CI matrix) guarantee that no + # script invocation in the process tree writes .specify/feature.json, even + # scripts that don't pass --no-persist themselves (#4128). + if [[ "${SPECIFY_FEATURE_NO_PERSIST:-}" == "1" || "${SPECIFY_FEATURE_NO_PERSIST:-}" == "true" ]]; then + no_persist=true + fi + + # Split decl/assignment so a SPECIFY_INIT_DIR validation failure in + # get_repo_root propagates as a hard error instead of being masked by `local`. + local repo_root + repo_root=$(get_repo_root) || return 1 + local current_branch + current_branch=$(get_current_branch) + + # Resolve feature directory. Priority: + # 1. SPECIFY_FEATURE_DIRECTORY env var (explicit override) + # 2. .specify/feature.json "feature_directory" key (persisted by specify command) + # 3. Error — no feature context available + local feature_dir + if [[ -n "${SPECIFY_FEATURE_DIRECTORY:-}" ]]; then + feature_dir="$SPECIFY_FEATURE_DIRECTORY" + # Normalize relative paths to absolute under repo root + [[ "$feature_dir" != /* ]] && feature_dir="$repo_root/$feature_dir" + # Persist to feature.json so future sessions without the env var still + # work — unless the caller opted out for read-only resolution (#3025). + if [[ "$no_persist" != true ]]; then + _persist_feature_json "$repo_root" "$SPECIFY_FEATURE_DIRECTORY" + fi + elif [[ -f "$repo_root/.specify/feature.json" ]]; then + local _fd + _fd=$(read_feature_json_feature_directory "$repo_root") + if [[ -n "$_fd" ]]; then + feature_dir="$_fd" + # Normalize relative paths to absolute under repo root + [[ "$feature_dir" != /* ]] && feature_dir="$repo_root/$feature_dir" + else + echo "ERROR: Feature directory not found. Set SPECIFY_FEATURE_DIRECTORY or ensure .specify/feature.json contains feature_directory." >&2 + return 1 + fi + else + echo "ERROR: Feature directory not found. Set SPECIFY_FEATURE_DIRECTORY or run the specify command to create .specify/feature.json." >&2 + return 1 + fi + + # When no branch context exists (no SPECIFY_FEATURE, feature resolved via + # SPECIFY_FEATURE_DIRECTORY or feature.json), fall back to the feature + # directory basename so CURRENT_BRANCH is a usable identifier rather than + # an empty, misleading value (issue #3026). + if [[ -z "$current_branch" ]]; then + local feature_dir_trimmed="${feature_dir%/}" + current_branch="${feature_dir_trimmed##*/}" + fi + + # Use printf '%q' to safely quote values, preventing shell injection + # via crafted branch names or paths containing special characters + printf 'REPO_ROOT=%q\n' "$repo_root" + printf 'CURRENT_BRANCH=%q\n' "$current_branch" + printf 'FEATURE_DIR=%q\n' "$feature_dir" + printf 'FEATURE_SPEC=%q\n' "$feature_dir/spec.md" + printf 'IMPL_PLAN=%q\n' "$feature_dir/plan.md" + printf 'TASKS=%q\n' "$feature_dir/tasks.md" + printf 'RESEARCH=%q\n' "$feature_dir/research.md" + printf 'DATA_MODEL=%q\n' "$feature_dir/data-model.md" + printf 'QUICKSTART=%q\n' "$feature_dir/quickstart.md" + printf 'CONTRACTS_DIR=%q\n' "$feature_dir/contracts" +} + +# Check if jq is available for safe JSON construction +has_jq() { + command -v jq >/dev/null 2>&1 +} + +get_invoke_separator() { + local repo_root="${1:-$(get_repo_root)}" + if [[ "${_SPECIFY_INVOKE_SEPARATOR_CACHE_REPO_ROOT:-}" == "$repo_root" && -n "${_SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE:-}" ]]; then + printf '%s\n' "$_SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE" + return 0 + fi + + local integration_json="$repo_root/.specify/integration.json" + local separator="." + local parsed=0 + + if [[ -f "$integration_json" ]]; then + # Try parsers in order (jq -> python3 -> awk), falling through on + # failure. Selection is by *parse success*, not mere availability: on + # Windows `python3` commonly resolves to the Microsoft Store App + # Execution Alias stub, which passes `command -v` but fails at runtime + # (exit 49). An availability-gated branch would pick python3, swallow + # its failure, and — because this function historically had no text + # fallback — silently return "." even for `-`-separator integrations + # (e.g. forge, cline), yielding wrong command hints (issue #3304). + if command -v jq >/dev/null 2>&1; then + local jq_separator + if jq_separator=$(jq -r '(.default_integration // .integration // "") as $k | if $k == "" then "." else (.integration_settings[$k].invoke_separator // ".") end' "$integration_json" 2>/dev/null); then + case "$jq_separator" in + "."|"-") separator="$jq_separator"; parsed=1 ;; + esac + fi + fi + + if [[ "$parsed" -eq 0 ]] && command -v python3 >/dev/null 2>&1; then + local py_separator + if py_separator=$(python3 - "$integration_json" <<'PY' 2>/dev/null +import json +import sys + +try: + with open(sys.argv[1], encoding="utf-8") as fh: + state = json.load(fh) + key = state.get("default_integration") or state.get("integration") or "" + settings = state.get("integration_settings") + separator = "." + if isinstance(key, str) and isinstance(settings, dict): + entry = settings.get(key) + if isinstance(entry, dict) and entry.get("invoke_separator") in {".", "-"}: + separator = entry["invoke_separator"] + print(separator) +except Exception: + sys.exit(1) +PY +); then + case "$py_separator" in + "."|"-") separator="$py_separator"; parsed=1 ;; + esac + fi + fi + + if [[ "$parsed" -eq 0 ]]; then + # Last-resort text fallback for environments with neither jq nor a + # working python3 (e.g. stock Windows + Git Bash). Reads the active + # integration key (default_integration, else integration) and its + # invoke_separator from within the integration_settings object. + # Handles both pretty-printed (the written form) and compact JSON. + # Accumulate all lines into one buffer in END rather than using + # gawk-only whole-file slurp (RS="^$"), so this stays portable to + # the BSD awk on macOS. + local awk_separator + awk_separator=$(awk ' + function keyval(d, name, v) { + if (match(d, "\"" name "\"[ \t\r\n]*:[ \t\r\n]*\"[^\"]*\"")) { + v=substr(d,RSTART,RLENGTH); sub(/^.*:[ \t\r\n]*"/,"",v); sub(/"$/,"",v); return v + } + return "" + } + { doc = doc $0 "\n" } + END { + key=keyval(doc,"default_integration"); if (key=="") key=keyval(doc,"integration") + sep="." + if (key!="") { + settings=doc + if (match(doc, /"integration_settings"[ \t\r\n]*:[ \t\r\n]*[{]/)) { + settings=substr(doc, RSTART+RLENGTH-1) + } + if (match(settings, "\"" key "\"[ \t\r\n]*:[ \t\r\n]*[{]")) { + start=RSTART+RLENGTH-1 + depth=0 + obj="" + for (i=start; i<=length(settings); i++) { + c=substr(settings,i,1) + obj=obj c + if (c=="{") depth++ + else if (c=="}") { depth--; if (depth==0) break } + } + if (match(obj, /"invoke_separator"[ \t\r\n]*:[ \t\r\n]*"[-.]"/)) { + tok=substr(obj,RSTART,RLENGTH); s=substr(tok,length(tok)-1,1) + if (s=="." || s=="-") sep=s + } + } + } + print sep + } + ' "$integration_json" 2>/dev/null) + case "$awk_separator" in + "."|"-") separator="$awk_separator" ;; + esac + fi + fi + + _SPECIFY_INVOKE_SEPARATOR_CACHE_REPO_ROOT="$repo_root" + _SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE="$separator" + printf '%s\n' "$separator" +} + +format_speckit_command() { + local command_name="$1" + local repo_root="${2:-$(get_repo_root)}" + local separator + if [[ "${_SPECIFY_INVOKE_SEPARATOR_CACHE_REPO_ROOT:-}" == "$repo_root" && -n "${_SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE:-}" ]]; then + separator="$_SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE" + else + separator=$(get_invoke_separator "$repo_root") + _SPECIFY_INVOKE_SEPARATOR_CACHE_REPO_ROOT="$repo_root" + _SPECIFY_INVOKE_SEPARATOR_CACHE_VALUE="$separator" + fi + + command_name="${command_name#/}" + command_name="${command_name#speckit.}" + command_name="${command_name#speckit-}" + command_name="${command_name//./$separator}" + + printf '/speckit%s%s\n' "$separator" "$command_name" +} + +# Escape a string for safe embedding in a JSON value (fallback when jq is unavailable). +# Handles backslash, double-quote, and JSON-required control character escapes (RFC 8259). +json_escape() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//$'\n'/\\n}" + s="${s//$'\t'/\\t}" + s="${s//$'\r'/\\r}" + s="${s//$'\b'/\\b}" + s="${s//$'\f'/\\f}" + # Escape any remaining U+0001-U+001F control characters as \uXXXX. + # (U+0000/NUL cannot appear in bash strings and is excluded.) + # LC_ALL=C ensures ${#s} counts bytes and ${s:$i:1} yields single bytes, + # so multi-byte UTF-8 sequences (first byte >= 0xC0) pass through intact. + local LC_ALL=C + local i char code + for (( i=0; i<${#s}; i++ )); do + char="${s:$i:1}" + printf -v code '%d' "'$char" 2>/dev/null || code=256 + if (( code >= 1 && code <= 31 )); then + printf '\\u%04x' "$code" + else + printf '%s' "$char" + fi + done +} + +check_file() { [[ -f "$1" ]] && echo " ✓ $2" || echo " ✗ $2"; } +check_dir() { [[ -d "$1" && -n $(ls -A "$1" 2>/dev/null) ]] && echo " ✓ $2" || echo " ✗ $2"; } + +_python3_command() { + if command -v python3 >/dev/null 2>&1 && + python3 -c 'import sys; raise SystemExit(sys.version_info.major != 3)' >/dev/null 2>&1; then + printf '%s\n' "python3" + elif command -v python >/dev/null 2>&1 && + python -c 'import sys; raise SystemExit(sys.version_info.major != 3)' >/dev/null 2>&1; then + printf '%s\n' "python" + elif command -v py >/dev/null 2>&1 && + py -3 -c 'import sys' >/dev/null 2>&1; then + printf '%s\n' "py -3" + else + return 1 + fi +} + +_sorted_extension_ids() { + local ext_dir="$1" + local python_spec + if python_spec=$(_python3_command); then + local -a python_cmd + read -r -a python_cmd <<< "$python_spec" + local py_stderr sorted_ids + py_stderr=$(mktemp) + if sorted_ids=$(SPECKIT_EXTENSIONS="$ext_dir" "${python_cmd[@]}" -c " +import json, os, re, sys +from pathlib import Path + +root = Path(os.environ['SPECKIT_EXTENSIONS']) +registered = {} +registry = root / '.registry' +if os.path.lexists(registry): + if not registry.is_file(): + print('registry_invalid: not a regular file', file=sys.stderr) + sys.exit(1) + try: + data = json.loads(registry.read_text(encoding='utf-8')) + except Exception as exc: + print('registry_invalid: ' + str(exc), file=sys.stderr) + sys.exit(1) + if not isinstance(data, dict): + print('registry_invalid: root must be a mapping', file=sys.stderr) + sys.exit(1) + raw_extensions = data.get('extensions', {}) + if not isinstance(raw_extensions, dict): + print('registry_invalid: extensions must be a mapping', file=sys.stderr) + sys.exit(1) + registered = raw_extensions + +def priority(value): + if isinstance(value, bool): + return 10 + try: + parsed = int(value) + return parsed if parsed >= 1 else 10 + except (TypeError, ValueError, OverflowError): + return 10 + +ranked = [] +for ext_id, meta in registered.items(): + if isinstance(ext_id, str) and re.fullmatch(r'[a-z0-9-]+', ext_id) and isinstance(meta, dict) and bool(meta.get('enabled', True)): + ranked.append((priority(meta.get('priority')), ext_id)) +for path in root.iterdir(): + if path.is_dir() and re.fullmatch(r'[a-z0-9-]+', path.name) and path.name not in registered: + ranked.append((10, path.name)) +for _, ext_id in sorted(ranked): + print(ext_id) +" 2>"$py_stderr"); then + rm -f "$py_stderr" + printf '%s\n' "$sorted_ids" + return 0 + else + echo "Error: invalid extension registry $ext_dir/.registry" >&2 + rm -f "$py_stderr" + return 1 + fi + fi + + if [ -e "$ext_dir/.registry" ] || [ -L "$ext_dir/.registry" ]; then + if [ ! -f "$ext_dir/.registry" ] || [ ! -r "$ext_dir/.registry" ]; then + echo "Error: invalid extension registry $ext_dir/.registry" >&2 + return 1 + fi + echo "Error: Python 3 is required to honor the extension registry" >&2 + return 2 + fi + + local ext extension_id + for ext in "$ext_dir"/*/; do + [ -d "$ext" ] || continue + extension_id=$(basename "$ext") + case "$extension_id" in *[!a-z0-9-]*) continue ;; esac + printf '%s\n' "$extension_id" + done +} + +# Resolve a template name to a file path using the priority stack: +# 1. .specify/templates/overrides/ +# 2. .specify/presets//templates/ (sorted by priority from .registry) +# 3. .specify/extensions//templates/ +# 4. .specify/templates/ (core) +resolve_template() { + local template_name="$1" + local repo_root="$2" + local base="$repo_root/.specify/templates" + + case "$template_name" in ""|*[!a-z0-9-]*) return 1 ;; esac + + # Priority 1: Project overrides + local override="$base/overrides/${template_name}.md" + [ -f "$override" ] && echo "$override" && return 0 + + # Priority 2: Installed presets (sorted by priority from .registry) + local presets_dir="$repo_root/.specify/presets" + if [ -d "$presets_dir" ]; then + local registry_file="$presets_dir/.registry" + local python_spec="" + local -a python_cmd=() + if python_spec=$(_python3_command); then + read -r -a python_cmd <<< "$python_spec" + fi + if [ -f "$registry_file" ] && [ "${#python_cmd[@]}" -gt 0 ]; then + # Read preset IDs sorted by priority (lower number = higher precedence). + # The python3 call is wrapped in an if-condition so that set -e does not + # abort the function when python3 exits non-zero (e.g. invalid JSON). + local sorted_presets="" + if sorted_presets=$(SPECKIT_REGISTRY="$registry_file" "${python_cmd[@]}" -c " +import json, re, sys, os +try: + with open(os.environ['SPECKIT_REGISTRY'], encoding='utf-8') as f: + data = json.load(f) + presets = data.get('presets', {}) + def priority(meta): + if not isinstance(meta, dict) or isinstance(meta.get('priority'), bool): + return 10 + try: + value = int(meta.get('priority', 10)) + return value if value >= 1 else 10 + except (TypeError, ValueError, OverflowError): + return 10 + for pid, meta in sorted(presets.items(), key=lambda x: (priority(x[1]), x[0])): + if isinstance(meta, dict) and bool(meta.get('enabled', True)) and re.fullmatch(r'[a-z0-9-]+', pid): + print(pid) +except Exception: + sys.exit(1) +" 2>/dev/null); then + if [ -n "$sorted_presets" ]; then + # python3 succeeded and returned preset IDs — search in priority order + while IFS= read -r preset_id; do + local candidate="$presets_dir/$preset_id/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + candidate="$presets_dir/$preset_id/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done <<< "$sorted_presets" + fi + # python3 succeeded but registry has no presets — nothing to search + else + # python3 failed (missing, or registry parse error) — fall back to unordered directory scan + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + candidate="$preset/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + else + # Fallback: alphabetical directory order (no python3 available) + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + candidate="$preset/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + fi + + # Priority 3: Extension-provided templates + local ext_dir="$repo_root/.specify/extensions" + if [ -d "$ext_dir" ]; then + local sorted_extensions="" + if ! sorted_extensions=$(_sorted_extension_ids "$ext_dir"); then + return 2 + fi + while IFS= read -r extension_id; do + [ -n "$extension_id" ] || continue + local ext="$ext_dir/$extension_id" + local candidate="$ext/templates/${template_name}.md" + [ -f "$candidate" ] || candidate="$ext/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done <<< "$sorted_extensions" + fi + + # Priority 4: Core templates + local core="$base/${template_name}.md" + [ -f "$core" ] && echo "$core" && return 0 + + # Template not found in any location. + # Return 1 so callers can distinguish "not found" from "found". + # Callers running under set -e should use: TEMPLATE=$(resolve_template ...) || true + return 1 +} + +# Resolve a template name to composed content using composition strategies. +# Reads strategy metadata from preset manifests and composes content +# from multiple layers using prepend, append, or wrap strategies. +# +# Usage: CONTENT=$(resolve_template_content "template-name" "$REPO_ROOT") +# Returns composed content string on stdout; exit code 1 if not found. +resolve_template_content() { + local template_name="$1" + local repo_root="$2" + local base="$repo_root/.specify/templates" + + case "$template_name" in ""|*[!a-z0-9-]*) return 1 ;; esac + + # Collect all layers (highest priority first) + local -a layer_paths=() + local -a layer_strategies=() + + # Priority 1: Project overrides (always "replace") + local override="$base/overrides/${template_name}.md" + if [ -f "$override" ]; then + if ! cat "$override"; then + echo "Error: failed to read template layer $override" >&2 + return 2 + fi + return 0 + fi + + local effective_base_found=false + + # Priority 2: Installed presets (sorted by priority from .registry) + local presets_dir="$repo_root/.specify/presets" + if [ -d "$presets_dir" ]; then + local registry_file="$presets_dir/.registry" + local sorted_presets="" + local registry_parsed=false + local python_spec="" + local -a python_cmd=() + if python_spec=$(_python3_command); then + read -r -a python_cmd <<< "$python_spec" + fi + if [ -f "$registry_file" ] && [ "${#python_cmd[@]}" -gt 0 ]; then + if sorted_presets=$(SPECKIT_REGISTRY="$registry_file" "${python_cmd[@]}" -c " +import json, re, sys, os +try: + with open(os.environ['SPECKIT_REGISTRY'], encoding='utf-8') as f: + data = json.load(f) + presets = data.get('presets', {}) + def priority(meta): + if not isinstance(meta, dict) or isinstance(meta.get('priority'), bool): + return 10 + try: + value = int(meta.get('priority', 10)) + return value if value >= 1 else 10 + except (TypeError, ValueError, OverflowError): + return 10 + for pid, meta in sorted(presets.items(), key=lambda x: (priority(x[1]), x[0])): + if isinstance(meta, dict) and bool(meta.get('enabled', True)) and re.fullmatch(r'[a-z0-9-]+', pid): + print(pid) +except Exception: + sys.exit(1) +" 2>/dev/null); then + registry_parsed=true + fi + fi + if [ "$registry_parsed" = false ]; then + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local fallback_id + fallback_id=$(basename "$preset") + case "$fallback_id" in *[!a-z0-9-]*) continue ;; esac + sorted_presets+="${sorted_presets:+$'\n'}$fallback_id" + done + fi + + if [ -n "$sorted_presets" ]; then + while IFS= read -r preset_id; do + local strategy="replace" + local manifest_file="" + local manifest="$presets_dir/$preset_id/preset.yml" + local manifest_declared=false + if [ -f "$manifest" ]; then + if [ "${#python_cmd[@]}" -eq 0 ]; then + echo "Error: Python 3 and PyYAML are required to resolve preset template composition" >&2 + return 2 + fi + local result + local py_stderr + local parse_status + py_stderr=$(mktemp) + if result=$(SPECKIT_MANIFEST="$manifest" SPECKIT_TMPL="$template_name" "${python_cmd[@]}" -c " +import sys, os +try: + import yaml +except ImportError: + print('yaml_missing', file=sys.stderr) + sys.exit(2) +try: + with open(os.environ['SPECKIT_MANIFEST'], encoding='utf-8') as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise ValueError('manifest root must be a mapping') + if 'provides' not in data: + raise ValueError('manifest missing provides section') + provides = data['provides'] + if not isinstance(provides, dict): + raise ValueError('manifest provides must be a mapping') + if 'templates' not in provides: + raise ValueError('manifest provides missing templates') + templates = provides['templates'] + if not isinstance(templates, list): + raise ValueError('manifest templates must be a list') + if not templates: + raise ValueError('manifest must provide at least one template') + valid_types = ('template', 'command', 'script') + valid_strategies = ('replace', 'prepend', 'append', 'wrap') + for t in templates: + if not isinstance(t, dict): + raise ValueError('manifest template entries must be mappings') + if 'type' not in t or 'name' not in t or 'file' not in t: + raise ValueError('manifest template entry missing type, name, or file') + for field in ('type', 'name', 'file'): + if not isinstance(t[field], str): + raise ValueError('manifest template ' + field + ' must be a string') + if t['type'] not in valid_types: + raise ValueError('invalid manifest template type') + strategy = t.get('strategy', 'replace') + if not isinstance(strategy, str): + raise ValueError('manifest template strategy must be a string') + strategy = strategy.lower() + if strategy not in valid_strategies: + raise ValueError('invalid manifest template strategy') + if t['type'] == 'script' and strategy not in ('replace', 'wrap'): + raise ValueError('invalid manifest script strategy') + for t in templates: + if t.get('name') == os.environ['SPECKIT_TMPL'] and t.get('type', 'template') == 'template': + file_value = t.get('file', '') + strategy = t.get('strategy', 'replace') + print('found\t' + strategy + '\t' + file_value) + sys.exit(0) + print('absent\treplace\t') +except Exception as exc: + print(f'manifest_invalid: {exc}', file=sys.stderr) + sys.exit(3) +" 2>"$py_stderr"); then + parse_status=0 + else + parse_status=$? + fi + if [ "$parse_status" -ne 0 ]; then + if [ "$parse_status" -eq 2 ]; then + echo "Error: PyYAML is required to resolve preset template composition" >&2 + else + echo "Error: invalid preset manifest $manifest" >&2 + fi + rm -f "$py_stderr" + return 2 + fi + if [ -n "$result" ]; then + local declaration + IFS=$'\t' read -r declaration strategy manifest_file <<< "$result" + [ "$declaration" = "found" ] && manifest_declared=true + strategy=$(printf '%s' "$strategy" | tr '[:upper:]' '[:lower:]') + fi + rm -f "$py_stderr" + fi + + local candidate="" + if [ -n "$manifest_file" ]; then + case "$manifest_file" in + /*|*../*) manifest_file="" ;; + esac + fi + if [ -n "$manifest_file" ]; then + local mf="$presets_dir/$preset_id/$manifest_file" + [ -f "$mf" ] && candidate="$mf" + fi + if [ -z "$candidate" ] && [ "$manifest_declared" = false ]; then + local cf="$presets_dir/$preset_id/templates/${template_name}.md" + [ -f "$cf" ] && candidate="$cf" + if [ -z "$candidate" ]; then + cf="$presets_dir/$preset_id/${template_name}.md" + [ -f "$cf" ] && candidate="$cf" + fi + fi + if [ -n "$candidate" ]; then + layer_paths+=("$candidate") + layer_strategies+=("$strategy") + if [ "$strategy" = "replace" ]; then + effective_base_found=true + break + fi + fi + done <<< "$sorted_presets" + fi + fi + + # Priority 3: Extension-provided templates (always "replace") + local ext_dir="$repo_root/.specify/extensions" + if [ "$effective_base_found" = false ] && [ -d "$ext_dir" ]; then + local sorted_extensions="" + if ! sorted_extensions=$(_sorted_extension_ids "$ext_dir"); then + return 2 + fi + while IFS= read -r extension_id; do + [ -n "$extension_id" ] || continue + local ext="$ext_dir/$extension_id" + local candidate="$ext/templates/${template_name}.md" + [ -f "$candidate" ] || candidate="$ext/${template_name}.md" + if [ -f "$candidate" ]; then + layer_paths+=("$candidate") + layer_strategies+=("replace") + effective_base_found=true + break + fi + done <<< "$sorted_extensions" + fi + + # Priority 4: Core templates (always "replace") + local core="$base/${template_name}.md" + if [ "$effective_base_found" = false ] && [ -f "$core" ]; then + layer_paths+=("$core") + layer_strategies+=("replace") + fi + + local count=${#layer_paths[@]} + [ "$count" -eq 0 ] && return 1 + + # Check if any layer uses a non-replace strategy + local has_composition=false + for s in "${layer_strategies[@]}"; do + [ "$s" != "replace" ] && has_composition=true && break + done + + # If the top (highest-priority) layer is replace, it wins entirely — + # lower layers are irrelevant regardless of their strategies. + if [ "${layer_strategies[0]}" = "replace" ]; then + if ! cat "${layer_paths[0]}"; then + echo "Error: failed to read template layer ${layer_paths[0]}" >&2 + return 2 + fi + return 0 + fi + + if [ "$has_composition" = false ]; then + if ! cat "${layer_paths[0]}"; then + echo "Error: failed to read template layer ${layer_paths[0]}" >&2 + return 2 + fi + return 0 + fi + + # Find the effective base: scan from highest priority (index 0) downward + # to find the nearest replace layer. Only compose layers above that base. + local base_idx=-1 + local i + for (( i=0; i&2 + return 2 + fi + + # Read the base content; compose layers above the base (higher priority) + local content + if ! content=$(cat "${layer_paths[$base_idx]}"; status=$?; printf x; exit "$status"); then + echo "Error: failed to read template layer ${layer_paths[$base_idx]}" >&2 + return 2 + fi + content="${content%x}" + + for (( i=base_idx-1; i>=0; i-- )); do + local path="${layer_paths[$i]}" + local strat="${layer_strategies[$i]}" + local layer_content + # Preserve trailing newlines + if ! layer_content=$(cat "$path"; status=$?; printf x; exit "$status"); then + echo "Error: failed to read template layer $path" >&2 + return 2 + fi + layer_content="${layer_content%x}" + + case "$strat" in + replace) content="$layer_content" ;; + prepend) + content=$(printf '%s\n\n%s' "$layer_content" "$content"; printf x) + content="${content%x}" + ;; + append) + content=$(printf '%s\n\n%s' "$content" "$layer_content"; printf x) + content="${content%x}" + ;; + wrap) + case "$layer_content" in + *'{CORE_TEMPLATE}'*) ;; + *) echo "Error: wrap strategy missing {CORE_TEMPLATE} placeholder" >&2; return 2 ;; + esac + # Consume the wrapper left to right instead of rewriting it in + # place. Rewriting re-scanned the string just modified, so base + # content holding a literal {CORE_TEMPLATE} reintroduced the + # token every pass and the loop never terminated. Advancing over + # ``rest`` bounds the work by the tokens in the original wrapper + # and leaves inserted content untouched, matching the single-pass + # semantics of .Replace()/.replace() in the PowerShell and Python + # ports. + local wrapped="" rest="$layer_content" + while [[ "$rest" == *'{CORE_TEMPLATE}'* ]]; do + wrapped="${wrapped}${rest%%\{CORE_TEMPLATE\}*}${content}" + rest="${rest#*\{CORE_TEMPLATE\}}" + done + content="${wrapped}${rest}" + ;; + *) echo "Error: unknown strategy '$strat'" >&2; return 2 ;; + esac + done + + printf '%s' "$content" + return 0 +} diff --git a/.specify/scripts/bash/create-new-feature.sh b/.specify/scripts/bash/create-new-feature.sh new file mode 100755 index 00000000..13670149 --- /dev/null +++ b/.specify/scripts/bash/create-new-feature.sh @@ -0,0 +1,427 @@ +#!/usr/bin/env bash + +set -e + +JSON_MODE=false +DRY_RUN=false +ALLOW_EXISTING=false +SHORT_NAME="" +BRANCH_NUMBER="" +USE_TIMESTAMP=false +NUMBER_EXPLICIT=false +ARGS=() +i=1 +while [ $i -le $# ]; do + arg="${!i}" + case "$arg" in + --json) + JSON_MODE=true + ;; + --dry-run) + DRY_RUN=true + ;; + --allow-existing-branch) + ALLOW_EXISTING=true + ;; + --short-name) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + # Check if the next argument is another option (starts with --) + if [[ "$next_arg" == --* ]]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + SHORT_NAME="$next_arg" + ;; + --number) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + if [[ "$next_arg" == --* ]]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + BRANCH_NUMBER="$next_arg" + if [ -n "$BRANCH_NUMBER" ]; then + NUMBER_EXPLICIT=true + fi + ;; + --timestamp) + USE_TIMESTAMP=true + ;; + --help|-h) + echo "Usage: $0 [--json] [--dry-run] [--allow-existing-branch] [--short-name ] [--number N] [--timestamp] " + echo "" + echo "Options:" + echo " --json Output in JSON format" + echo " --dry-run Compute feature name and paths without creating directories or files" + echo " --allow-existing-branch Reuse an existing feature directory if it already exists" + echo " --short-name Provide a custom short name (2-4 words) for the feature" + echo " --number N Prefer a feature number (auto-corrected if its specs prefix exists)" + echo " --timestamp Use timestamp prefix (YYYYMMDD-HHMMSS) instead of sequential numbering" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 'Add user authentication system' --short-name 'user-auth'" + echo " $0 'Implement OAuth2 integration for API' --number 5" + echo " $0 --timestamp --short-name 'user-auth' 'Add user authentication'" + exit 0 + ;; + *) + ARGS+=("$arg") + ;; + esac + i=$((i + 1)) +done + +FEATURE_DESCRIPTION="${ARGS[*]}" +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Usage: $0 [--json] [--dry-run] [--allow-existing-branch] [--short-name ] [--number N] [--timestamp] " >&2 + exit 1 +fi + +# Trim whitespace and validate description is not empty (e.g., user passed only whitespace) +FEATURE_DESCRIPTION=$(echo "$FEATURE_DESCRIPTION" | sed -E 's/^[[:space:]]+|[[:space:]]+$//g') +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Error: Feature description cannot be empty or contain only whitespace" >&2 + exit 1 +fi + +MAX_FEATURE_NUMBER=9223372036854775807 +MAX_BRANCH_LENGTH=244 + +is_feature_number_in_range() { + local value="$1" + local normalized="${value#"${value%%[!0]*}"}" + [ -n "$normalized" ] || normalized=0 + [ ${#normalized} -lt ${#MAX_FEATURE_NUMBER} ] && return 0 + [ ${#normalized} -gt ${#MAX_FEATURE_NUMBER} ] && return 1 + # Equal-length digit strings must be compared without arithmetic overflow. + # shellcheck disable=SC2071 + [[ "$normalized" < "$MAX_FEATURE_NUMBER" || "$normalized" == "$MAX_FEATURE_NUMBER" ]] +} + +# Function to get highest number from specs directory +get_highest_from_specs() { + local specs_dir="$1" + local highest=0 + + if [ -d "$specs_dir" ]; then + for dir in "$specs_dir"/*; do + [ -d "$dir" ] || continue + dirname=$(basename "$dir") + # Match sequential prefixes (>=3 digits), but skip timestamp dirs. + if echo "$dirname" | grep -Eq '^[0-9]{3,}-' && ! echo "$dirname" | grep -Eq '^[0-9]{8}-[0-9]{6}-'; then + number=$(echo "$dirname" | grep -Eo '^[0-9]+') + if is_feature_number_in_range "$number"; then + number=$((10#$number)) + if [ "$number" -gt "$highest" ]; then + highest=$number + fi + fi + fi + done + fi + + echo "$highest" +} + +# Return success when a spec directory owns the given numeric prefix. +spec_prefix_exists() { + local specs_dir="$1" + local feature_num="$2" + + for spec_path in "$specs_dir/${feature_num}-"*; do + [ -d "$spec_path" ] && return 0 + done + return 1 +} + +# Function to clean and format a branch name +# +# Three details keep this byte-identical to the Python and PowerShell twins: +# * LC_ALL=C -- in a UTF-8 locale glibc resolves the a-z *range* through +# collation, so [^a-z0-9] keeps accented lowercase letters that +# re.sub(r"[^a-z0-9]", ...) and .NET's -replace both strip. +# * `--*` instead of the GNU-only `\+`, which POSIX/BSD sed reads as a literal +# '+', leaving repeated separators uncollapsed on macOS. +# * printf instead of echo, so a name of "-n"/"-e"/"-E" is text, not options. +clean_branch_name() { + local name="$1" + local -x LC_ALL=C + printf '%s\n' "$name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//' +} + +# Fit a feature prefix and suffix within GitHub's branch-name limit. +fit_branch_name() { + local feature_num="$1" + local branch_suffix="$2" + local branch_name="${feature_num}-${branch_suffix}" + + if [ ${#branch_name} -gt $MAX_BRANCH_LENGTH ]; then + local prefix_length=$(( ${#feature_num} + 1 )) + local max_suffix_length=$((MAX_BRANCH_LENGTH - prefix_length)) + local truncated_suffix + truncated_suffix=$(printf '%s' "$branch_suffix" | cut -c "1-$max_suffix_length" | sed 's/-$//') + branch_name="${feature_num}-${truncated_suffix}" + fi + + printf '%s' "$branch_name" +} + +# Quote a value for POSIX shell reuse, byte-identical to Python's shlex.quote +# so the persistence hints match the Python variant exactly (printf %q output +# differs between bash versions and from shlex.quote for spaces/metachars). +shell_quote() { + local value="$1" LC_ALL=C + if [[ "$value" =~ ^[A-Za-z0-9_@%+=:,./-]+$ ]]; then + printf '%s' "$value" + else + local q="'\"'\"'" + printf "'%s'" "${value//\'/$q}" + fi +} + +# Resolve repository root using common.sh functions which prioritize .specify +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +REPO_ROOT=$(get_repo_root) || exit 1 + +cd "$REPO_ROOT" + +SPECS_DIR="$REPO_ROOT/specs" +if [ "$DRY_RUN" != true ]; then + mkdir -p "$SPECS_DIR" +fi + +# Function to generate branch name with stop word filtering and length filtering +generate_branch_name() { + local description="$1" + + # Common stop words to filter out + local stop_words="^(i|a|an|the|to|for|of|in|on|at|by|with|from|is|are|was|were|be|been|being|have|has|had|do|does|did|will|would|should|could|can|may|might|must|shall|this|that|these|those|my|your|our|their|want|need|add|get|set)$" + + # Convert to lowercase and split into words. LC_ALL=C for the same + # collation reason documented on clean_branch_name, and so the `grep -qw` + # acronym probe below uses ASCII word boundaries like the Python twin's + # (?= 3 OR are potential acronyms) + if ! echo "$word" | grep -qiE "$stop_words"; then + if [ ${#word} -ge 3 ]; then + meaningful_words+=("$word") + # Keep short words that appear as an uppercase acronym in the original. + # Uppercase via tr and match with grep -w (both portable) rather than + # bash's 4+ "^^" case expansion (breaks on macOS bash 3.2) and \b (non-POSIX). + elif printf '%s' "$description" | grep -qw -- "$(printf '%s' "$word" | tr '[:lower:]' '[:upper:]')"; then + meaningful_words+=("$word") + fi + fi + done + + # If we have meaningful words, use first 3-4 of them + if [ ${#meaningful_words[@]} -gt 0 ]; then + local max_words=3 + if [ ${#meaningful_words[@]} -eq 4 ]; then max_words=4; fi + + local result="" + local count=0 + for word in "${meaningful_words[@]}"; do + if [ $count -ge $max_words ]; then break; fi + if [ -n "$result" ]; then result="$result-"; fi + result="$result$word" + count=$((count + 1)) + done + echo "$result" + else + # Fallback to original logic if no meaningful words found + local cleaned=$(clean_branch_name "$description") + echo "$cleaned" | tr '-' '\n' | grep -v '^$' | head -3 | tr '\n' '-' | sed 's/-$//' + fi +} + +# Generate branch name +if [ -n "$SHORT_NAME" ]; then + # Use provided short name, just clean it up + BRANCH_SUFFIX=$(clean_branch_name "$SHORT_NAME") +else + # Generate from description with smart filtering + BRANCH_SUFFIX=$(generate_branch_name "$FEATURE_DESCRIPTION") +fi + +if [ -z "$BRANCH_SUFFIX" ]; then + echo "[specify] Warning: Feature name is empty after removing unsupported characters. Use --short-name with ASCII letters or digits (for example, user-auth)." >&2 +fi + +# Warn if --number and --timestamp are both specified +if [ "$USE_TIMESTAMP" = true ] && [ -n "$BRANCH_NUMBER" ]; then + >&2 echo "[specify] Warning: --number is ignored when --timestamp is used" + BRANCH_NUMBER="" +fi + +# Determine branch prefix +if [ "$USE_TIMESTAMP" = true ]; then + FEATURE_NUM=$(date +%Y%m%d-%H%M%S) + BRANCH_NAME="${FEATURE_NUM}-${BRANCH_SUFFIX}" +else + if [ -n "$BRANCH_NUMBER" ] && [[ ! "$BRANCH_NUMBER" =~ ^[0-9]+$ ]]; then + echo "Error: --number must be an unsigned integer, got '$BRANCH_NUMBER'" >&2 + exit 1 + fi + + # Bash arithmetic is signed 64-bit; reject digit strings that would wrap. + if [ -n "$BRANCH_NUMBER" ] && ! is_feature_number_in_range "$BRANCH_NUMBER"; then + echo "Error: --number must be between 0 and $MAX_FEATURE_NUMBER, got '$BRANCH_NUMBER'" >&2 + exit 1 + fi + + # Determine branch number from existing feature directories + if [ -z "$BRANCH_NUMBER" ]; then + HIGHEST=$(get_highest_from_specs "$SPECS_DIR") + if [ "$HIGHEST" -eq "$MAX_FEATURE_NUMBER" ]; then + echo "Error: feature number must be between 0 and $MAX_FEATURE_NUMBER, got '9223372036854775808'" >&2 + exit 1 + fi + BRANCH_NUMBER=$((HIGHEST + 1)) + fi + + # Force base-10 interpretation to prevent octal conversion (e.g., 010 → 8 in octal, but should be 10 in decimal) + FEATURE_NUM=$(printf "%03d" "$((10#$BRANCH_NUMBER))") + + # Treat an explicit number as a preference when its prefix is already used + # by a feature directory. Auto-detected numbers are already conflict-free. + if [ "$NUMBER_EXPLICIT" = true ]; then + SPEC_CONFLICT=false + REQUESTED_BRANCH_NAME=$(fit_branch_name "$FEATURE_NUM" "$BRANCH_SUFFIX") + REQUESTED_DIR="$SPECS_DIR/$REQUESTED_BRANCH_NAME" + if [ "$ALLOW_EXISTING" != true ] || [ ! -d "$REQUESTED_DIR" ]; then + spec_prefix_exists "$SPECS_DIR" "$FEATURE_NUM" && SPEC_CONFLICT=true + fi + + if [ "$SPEC_CONFLICT" = true ]; then + REQUESTED_NUM="$FEATURE_NUM" + HIGHEST=$(get_highest_from_specs "$SPECS_DIR") + BRANCH_NUMBER=$HIGHEST + while true; do + if [ "$BRANCH_NUMBER" -eq "$MAX_FEATURE_NUMBER" ]; then + echo "Error: feature number must be between 0 and $MAX_FEATURE_NUMBER, got '9223372036854775808'" >&2 + exit 1 + fi + BRANCH_NUMBER=$((BRANCH_NUMBER + 1)) + FEATURE_NUM=$(printf "%03d" "$((10#$BRANCH_NUMBER))") + spec_prefix_exists "$SPECS_DIR" "$FEATURE_NUM" || break + done + >&2 echo "[specify] Warning: --number $REQUESTED_NUM conflicts with an existing spec directory; using $FEATURE_NUM instead" + fi + fi + +fi + +# GitHub enforces a 244-byte limit on branch names +# Validate and truncate if necessary +ORIGINAL_BRANCH_NAME="${FEATURE_NUM}-${BRANCH_SUFFIX}" +BRANCH_NAME=$(fit_branch_name "$FEATURE_NUM" "$BRANCH_SUFFIX") +if [ "$BRANCH_NAME" != "$ORIGINAL_BRANCH_NAME" ]; then + >&2 echo "[specify] Warning: Branch name exceeded GitHub's 244-byte limit" + >&2 echo "[specify] Original: $ORIGINAL_BRANCH_NAME (${#ORIGINAL_BRANCH_NAME} bytes)" + >&2 echo "[specify] Truncated to: $BRANCH_NAME (${#BRANCH_NAME} bytes)" +fi + +FEATURE_DIR="$SPECS_DIR/$BRANCH_NAME" +SPEC_FILE="$FEATURE_DIR/spec.md" + +if [ "$DRY_RUN" != true ]; then + if [ -d "$FEATURE_DIR" ] && [ "$ALLOW_EXISTING" != true ]; then + if [ "$USE_TIMESTAMP" = true ]; then + >&2 echo "Error: Feature directory '$FEATURE_DIR' already exists. Rerun to get a new timestamp or use a different --short-name." + else + >&2 echo "Error: Feature directory '$FEATURE_DIR' already exists. Please use a different feature name or specify a different number with --number." + fi + exit 1 + fi + + NEEDS_SPEC=false + SPEC_TEMPLATE_FOUND=false + SPEC_TEMPLATE_CONTENT="" + if [ ! -f "$SPEC_FILE" ]; then + NEEDS_SPEC=true + if SPEC_TEMPLATE_CONTENT=$(resolve_template_content "spec-template" "$REPO_ROOT"; status=$?; printf x; exit "$status"); then + SPEC_TEMPLATE_CONTENT="${SPEC_TEMPLATE_CONTENT%x}" + SPEC_TEMPLATE_FOUND=true + else + resolve_status=$? + if [ "$resolve_status" -ne 1 ]; then + exit "$resolve_status" + fi + fi + fi + + mkdir -p "$FEATURE_DIR" + + if [ "$NEEDS_SPEC" = true ]; then + if [ "$SPEC_TEMPLATE_FOUND" = true ]; then + printf '%s' "$SPEC_TEMPLATE_CONTENT" > "$SPEC_FILE" + else + echo "Warning: Spec template not found; created empty spec file" >&2 + touch "$SPEC_FILE" + fi + fi + + # Persist to .specify/feature.json so downstream commands can find the + # feature, unless the orchestrator opted out via SPECIFY_FEATURE_NO_PERSIST (#4129). + if [[ "${SPECIFY_FEATURE_NO_PERSIST:-}" != "1" && "${SPECIFY_FEATURE_NO_PERSIST:-}" != "true" ]]; then + _persist_feature_json "$REPO_ROOT" "$FEATURE_DIR" + fi + + # Inform the user how to set feature state in their own shell + printf '# To persist: export SPECIFY_FEATURE=%s\n' "$(shell_quote "$BRANCH_NAME")" >&2 + printf '# export SPECIFY_FEATURE_DIRECTORY=%s\n' "$(shell_quote "$FEATURE_DIR")" >&2 +fi + +if $JSON_MODE; then + if command -v jq >/dev/null 2>&1; then + if [ "$DRY_RUN" = true ]; then + jq -cn \ + --arg branch_name "$BRANCH_NAME" \ + --arg spec_file "$SPEC_FILE" \ + --arg feature_num "$FEATURE_NUM" \ + '{BRANCH_NAME:$branch_name,SPEC_FILE:$spec_file,FEATURE_NUM:$feature_num,DRY_RUN:true}' + else + jq -cn \ + --arg branch_name "$BRANCH_NAME" \ + --arg spec_file "$SPEC_FILE" \ + --arg feature_num "$FEATURE_NUM" \ + '{BRANCH_NAME:$branch_name,SPEC_FILE:$spec_file,FEATURE_NUM:$feature_num}' + fi + else + if [ "$DRY_RUN" = true ]; then + printf '{"BRANCH_NAME":"%s","SPEC_FILE":"%s","FEATURE_NUM":"%s","DRY_RUN":true}\n' "$(json_escape "$BRANCH_NAME")" "$(json_escape "$SPEC_FILE")" "$(json_escape "$FEATURE_NUM")" + else + printf '{"BRANCH_NAME":"%s","SPEC_FILE":"%s","FEATURE_NUM":"%s"}\n' "$(json_escape "$BRANCH_NAME")" "$(json_escape "$SPEC_FILE")" "$(json_escape "$FEATURE_NUM")" + fi + fi +else + echo "BRANCH_NAME: $BRANCH_NAME" + echo "SPEC_FILE: $SPEC_FILE" + echo "FEATURE_NUM: $FEATURE_NUM" + if [ "$DRY_RUN" != true ]; then + printf '# To persist in your shell: export SPECIFY_FEATURE=%s\n' "$(shell_quote "$BRANCH_NAME")" + printf '# export SPECIFY_FEATURE_DIRECTORY=%s\n' "$(shell_quote "$FEATURE_DIR")" + fi +fi diff --git a/.specify/scripts/bash/resolve-template.sh b/.specify/scripts/bash/resolve-template.sh new file mode 100755 index 00000000..da05d2df --- /dev/null +++ b/.specify/scripts/bash/resolve-template.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR="$(CDPATH="" cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +JSON_MODE=false +TEMPLATE_NAME="" + +for arg in "$@"; do + case "$arg" in + --json) JSON_MODE=true ;; + --help|-h) + echo "Usage: $0 [--json]" + exit 0 + ;; + -*) + echo "ERROR: Unknown option '$arg'" >&2 + exit 1 + ;; + *) + if [[ -n "$TEMPLATE_NAME" ]]; then + echo "ERROR: Unexpected argument '$arg'" >&2 + exit 1 + fi + TEMPLATE_NAME="$arg" + ;; + esac +done + +if [[ -z "$TEMPLATE_NAME" ]]; then + echo "ERROR: Template name is required" >&2 + exit 1 +fi + +REPO_ROOT=$(get_repo_root) +if TEMPLATE_CONTENT=$(resolve_template_content "$TEMPLATE_NAME" "$REPO_ROOT"; status=$?; printf x; exit "$status"); then + TEMPLATE_CONTENT="${TEMPLATE_CONTENT%x}" +else + echo "ERROR: Could not resolve required $TEMPLATE_NAME from the template override stack for $REPO_ROOT" >&2 + exit 1 +fi + +if $JSON_MODE; then + if has_jq; then + jq -cn \ + --arg template_name "$TEMPLATE_NAME" \ + --arg template_content "$TEMPLATE_CONTENT" \ + '{TEMPLATE_NAME:$template_name,TEMPLATE_CONTENT:$template_content}' + else + printf '{"TEMPLATE_NAME":"%s","TEMPLATE_CONTENT":"%s"}\n' \ + "$(json_escape "$TEMPLATE_NAME")" "$(json_escape "$TEMPLATE_CONTENT")" + fi +else + printf '%s' "$TEMPLATE_CONTENT" +fi diff --git a/.specify/scripts/bash/setup-plan.sh b/.specify/scripts/bash/setup-plan.sh new file mode 100755 index 00000000..aa394183 --- /dev/null +++ b/.specify/scripts/bash/setup-plan.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +# Parse command line arguments +JSON_MODE=false + +for arg in "$@"; do + case "$arg" in + --json) + JSON_MODE=true + ;; + --help|-h) + echo "Usage: $0 [--json]" + echo " --json Output results in JSON format" + echo " --help Show this help message" + exit 0 + ;; + *) + echo "ERROR: Unknown option '$arg'" >&2 + exit 1 + ;; + esac +done + +# Get script directory and load common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get all paths and variables from common functions +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +# Ensure the feature directory exists +mkdir -p "$FEATURE_DIR" + +# Copy plan template if plan doesn't already exist +if [[ -f "$IMPL_PLAN" ]]; then + if $JSON_MODE; then + echo "Plan already exists at $IMPL_PLAN, skipping template copy" >&2 + else + echo "Plan already exists at $IMPL_PLAN, skipping template copy" + fi +else + if resolve_template_content "plan-template" "$REPO_ROOT" > "$IMPL_PLAN"; then + if $JSON_MODE; then + echo "Copied plan template to $IMPL_PLAN" >&2 + else + echo "Copied plan template to $IMPL_PLAN" + fi + else + resolve_status=$? + rm -f "$IMPL_PLAN" + if [ "$resolve_status" -ne 1 ]; then + exit "$resolve_status" + fi + if $JSON_MODE; then + echo "Warning: Plan template not found" >&2 + else + echo "Warning: Plan template not found" + fi + touch "$IMPL_PLAN" + fi +fi + +# Output results +if $JSON_MODE; then + if has_jq; then + jq -cn \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg feature_dir "$FEATURE_DIR" \ + --arg branch "$CURRENT_BRANCH" \ + '{FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,FEATURE_DIR:$feature_dir,BRANCH:$branch}' + else + printf '{"FEATURE_SPEC":"%s","IMPL_PLAN":"%s","FEATURE_DIR":"%s","BRANCH":"%s"}\n' \ + "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$CURRENT_BRANCH")" + fi +else + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "FEATURE_DIR: $FEATURE_DIR" + echo "BRANCH: $CURRENT_BRANCH" +fi diff --git a/.specify/scripts/bash/setup-tasks.sh b/.specify/scripts/bash/setup-tasks.sh new file mode 100755 index 00000000..0383f9f3 --- /dev/null +++ b/.specify/scripts/bash/setup-tasks.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -e + +# Parse command line arguments +JSON_MODE=false + +for arg in "$@"; do + case "$arg" in + --json) JSON_MODE=true ;; + --help|-h) + echo "Usage: $0 [--json]" + echo " --json Output results in JSON format" + echo " --help Show this help message" + exit 0 + ;; + *) echo "ERROR: Unknown option '$arg'" >&2; exit 1 ;; + esac +done + +# Source common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get feature paths +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +# Validate required files +if [[ ! -f "$IMPL_PLAN" ]]; then + echo "ERROR: plan.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit-plan first to create the implementation plan." >&2 + exit 1 +fi + +if [[ ! -f "$FEATURE_SPEC" ]]; then + echo "ERROR: spec.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit-specify first to create the feature structure." >&2 + exit 1 +fi + +# Build available docs list +docs=() +[[ -f "$RESEARCH" ]] && docs+=("research.md") +[[ -f "$DATA_MODEL" ]] && docs+=("data-model.md") +if [[ -d "$CONTRACTS_DIR" ]] && [[ -n "$(ls -A "$CONTRACTS_DIR" 2>/dev/null)" ]]; then + docs+=("contracts/") +fi +[[ -f "$QUICKSTART" ]] && docs+=("quickstart.md") + +# Resolve tasks template through override stack +TASKS_TEMPLATE=$(resolve_template "tasks-template" "$REPO_ROOT") || true +if TASKS_TEMPLATE_CONTENT=$(resolve_template_content "tasks-template" "$REPO_ROOT"; status=$?; printf x; exit "$status"); then + TASKS_TEMPLATE_CONTENT="${TASKS_TEMPLATE_CONTENT%x}" +else + echo "ERROR: Could not resolve required tasks-template from the template override stack for $REPO_ROOT" >&2 + echo "Template 'tasks-template' was not found in any supported location (overrides, presets, extensions, or shared core). Add an override at .specify/templates/overrides/tasks-template.md, or run 'specify init' / reinstall shared infra to restore the core .specify/templates/tasks-template.md template." >&2 + exit 1 +fi + +# Output results +if $JSON_MODE; then + if has_jq; then + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(printf '%s\n' "${docs[@]}" | jq -R . | jq -s .) + fi + jq -cn \ + --arg feature_dir "$FEATURE_DIR" \ + --argjson docs "$json_docs" \ + --arg tasks_template "${TASKS_TEMPLATE:-}" \ + --arg tasks_template_content "$TASKS_TEMPLATE_CONTENT" \ + '{FEATURE_DIR:$feature_dir,AVAILABLE_DOCS:$docs,TASKS_TEMPLATE:$tasks_template,TASKS_TEMPLATE_CONTENT:$tasks_template_content}' + else + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(for d in "${docs[@]}"; do printf '"%s",' "$(json_escape "$d")"; done) + json_docs="[${json_docs%,}]" + fi + printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s,"TASKS_TEMPLATE":"%s","TASKS_TEMPLATE_CONTENT":"%s"}\n' \ + "$(json_escape "$FEATURE_DIR")" "$json_docs" "$(json_escape "${TASKS_TEMPLATE:-}")" "$(json_escape "$TASKS_TEMPLATE_CONTENT")" + fi +else + echo "FEATURE_DIR: $FEATURE_DIR" + echo "TASKS_TEMPLATE: ${TASKS_TEMPLATE:-not found}" + echo "AVAILABLE_DOCS:" + check_file "$RESEARCH" "research.md" + check_file "$DATA_MODEL" "data-model.md" + check_dir "$CONTRACTS_DIR" "contracts/" + check_file "$QUICKSTART" "quickstart.md" +fi diff --git a/.specify/templates/checklist-template.md b/.specify/templates/checklist-template.md new file mode 100644 index 00000000..c4c4ffb4 --- /dev/null +++ b/.specify/templates/checklist-template.md @@ -0,0 +1,45 @@ +# [CHECKLIST TYPE] Checklist: [FEATURE NAME] + +**Purpose**: [Brief description of what this checklist covers] +**Created**: [DATE] +**Feature**: [Link to spec.md or relevant documentation] + +**Note**: This custom checklist is generated by the `/speckit-checklist` command based on feature context and requirements. +**Review Ownership**: This checklist is a reviewer-owned requirements-quality review artifact. Mark an item `[x]` only when the reviewer determines the requirements-quality criterion is satisfied. +**Marker Semantics**: `[x]` means the criterion has been reviewed and satisfied for requirements quality. It does not mean implementation work is complete. + + + +## [Category 1] + +- [ ] CHK001 First checklist item with clear action +- [ ] CHK002 Second checklist item +- [ ] CHK003 Third checklist item + +## [Category 2] + +- [ ] CHK004 Another category item +- [ ] CHK005 Item with specific criteria +- [ ] CHK006 Final item in this category + +## Notes + +- Mark items `[x]` only after review confirms the requirement-quality criterion is satisfied +- Leave items unchecked when they still require clarification, correction, or reviewer evaluation +- `/speckit-implement` reads checklist checkbox state as a gate and must not modify markers +- `checklists/requirements.md` has a separate built-in lifecycle maintained by `/speckit-specify` and `/speckit-clarify` +- Add comments or findings inline +- Link to relevant resources or documentation +- Items are numbered sequentially for easy reference diff --git a/.specify/templates/constitution-template.md b/.specify/templates/constitution-template.md new file mode 100644 index 00000000..a4670ff4 --- /dev/null +++ b/.specify/templates/constitution-template.md @@ -0,0 +1,50 @@ +# [PROJECT_NAME] Constitution + + +## Core Principles + +### [PRINCIPLE_1_NAME] + +[PRINCIPLE_1_DESCRIPTION] + + +### [PRINCIPLE_2_NAME] + +[PRINCIPLE_2_DESCRIPTION] + + +### [PRINCIPLE_3_NAME] + +[PRINCIPLE_3_DESCRIPTION] + + +### [PRINCIPLE_4_NAME] + +[PRINCIPLE_4_DESCRIPTION] + + +### [PRINCIPLE_5_NAME] + +[PRINCIPLE_5_DESCRIPTION] + + +## [SECTION_2_NAME] + + +[SECTION_2_CONTENT] + + +## [SECTION_3_NAME] + + +[SECTION_3_CONTENT] + + +## Governance + + +[GOVERNANCE_RULES] + + +**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE] + diff --git a/.specify/templates/plan-template.md b/.specify/templates/plan-template.md new file mode 100644 index 00000000..be1aa880 --- /dev/null +++ b/.specify/templates/plan-template.md @@ -0,0 +1,113 @@ +# Implementation Plan: [FEATURE] + +**Branch**: `[###-feature-name]` | **Date**: [DATE] | **Spec**: [link] + +**Input**: Feature specification from `/specs/[###-feature-name]/spec.md` + +**Note**: This template is filled in by the `/speckit-plan` command; its definition describes the execution workflow. + +## Summary + +[Extract from feature spec: primary requirement + technical approach from research] + +## Technical Context + + + +**Language/Version**: [e.g., Python 3.11, Swift 5.9, Rust 1.75 or NEEDS CLARIFICATION] + +**Primary Dependencies**: [e.g., FastAPI, UIKit, LLVM or NEEDS CLARIFICATION] + +**Storage**: [if applicable, e.g., PostgreSQL, CoreData, files or N/A] + +**Testing**: [e.g., pytest, XCTest, cargo test or NEEDS CLARIFICATION] + +**Target Platform**: [e.g., Linux server, iOS 15+, WASM or NEEDS CLARIFICATION] + +**Project Type**: [e.g., library/cli/web-service/mobile-app/compiler/desktop-app or NEEDS CLARIFICATION] + +**Performance Goals**: [domain-specific, e.g., 1000 req/s, 10k lines/sec, 60 fps or NEEDS CLARIFICATION] + +**Constraints**: [domain-specific, e.g., <200ms p95, <100MB memory, offline-capable or NEEDS CLARIFICATION] + +**Scale/Scope**: [domain-specific, e.g., 10k users, 1M LOC, 50 screens or NEEDS CLARIFICATION] + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +[Gates determined based on constitution file] + +## Project Structure + +### Documentation (this feature) + +```text +specs/[###-feature]/ +├── plan.md # This file (/speckit-plan command output) +├── research.md # Phase 0 output (/speckit-plan command) +├── data-model.md # Phase 1 output (/speckit-plan command) +├── quickstart.md # Phase 1 output (/speckit-plan command) +├── contracts/ # Phase 1 output (/speckit-plan command) +└── tasks.md # Phase 2 output (/speckit-tasks command - NOT created by /speckit-plan) +``` + +### Source Code (repository root) + + +```text +# [REMOVE IF UNUSED] Option 1: Single project (DEFAULT) +src/ +├── models/ +├── services/ +├── cli/ +└── lib/ + +tests/ +├── contract/ +├── integration/ +└── unit/ + +# [REMOVE IF UNUSED] Option 2: Web application (when "frontend" + "backend" detected) +backend/ +├── src/ +│ ├── models/ +│ ├── services/ +│ └── api/ +└── tests/ + +frontend/ +├── src/ +│ ├── components/ +│ ├── pages/ +│ └── services/ +└── tests/ + +# [REMOVE IF UNUSED] Option 3: Mobile + API (when "iOS/Android" detected) +api/ +└── [same as backend above] + +ios/ or android/ +└── [platform-specific structure: feature modules, UI flows, platform tests] +``` + +**Structure Decision**: [Document the selected structure and reference the real +directories captured above] + +## Complexity Tracking + +> **Fill ONLY if Constitution Check has violations that must be justified** + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| [e.g., 4th project] | [current need] | [why 3 projects insufficient] | +| [e.g., Repository pattern] | [specific problem] | [why direct DB access insufficient] | diff --git a/.specify/templates/spec-template.md b/.specify/templates/spec-template.md new file mode 100644 index 00000000..ceb28776 --- /dev/null +++ b/.specify/templates/spec-template.md @@ -0,0 +1,131 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` + +**Created**: [DATE] + +**Status**: Draft + +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + + + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + + + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + + + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] + +## Assumptions + + + +- [Assumption about target users, e.g., "Users have stable internet connectivity"] +- [Assumption about scope boundaries, e.g., "Mobile support is out of scope for v1"] +- [Assumption about data/environment, e.g., "Existing authentication system will be reused"] +- [Dependency on existing system/service, e.g., "Requires access to the existing user profile API"] diff --git a/.specify/templates/tasks-template.md b/.specify/templates/tasks-template.md new file mode 100644 index 00000000..d46a1f1f --- /dev/null +++ b/.specify/templates/tasks-template.md @@ -0,0 +1,252 @@ +--- + +description: "Task list template for feature implementation" +--- + +# Tasks: [FEATURE NAME] + +**Input**: Design documents from `/specs/[###-feature-name]/` + +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: The examples below include test tasks. Tests are OPTIONAL - only include them if explicitly requested in the feature specification. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Path Conventions + +- **Single project**: `src/`, `tests/` at repository root +- **Web app**: `backend/src/`, `frontend/src/` +- **Mobile**: `api/src/`, `ios/src/` or `android/src/` +- Paths shown below assume single project - adjust based on plan.md structure + + + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Project initialization and basic structure + +- [ ] T001 Create project structure per implementation plan +- [ ] T002 Initialize [language] project with [framework] dependencies +- [ ] T003 [P] Configure linting and formatting tools + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete + +Examples of foundational tasks (adjust based on your project): + +- [ ] T004 Setup database schema and migrations framework +- [ ] T005 [P] Implement authentication/authorization framework +- [ ] T006 [P] Setup API routing and middleware structure +- [ ] T007 Create base models/entities that all stories depend on +- [ ] T008 Configure error handling and logging infrastructure +- [ ] T009 Setup environment configuration management + +**Checkpoint**: Foundation ready - user story implementation can now begin in parallel + +--- + +## Phase 3: User Story 1 - [Title] (Priority: P1) 🎯 MVP + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 1 (OPTIONAL - only if tests requested) ⚠️ + +> **NOTE: Write these tests FIRST, ensure they FAIL before implementation** + +- [ ] T010 [P] [US1] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T011 [P] [US1] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 1 + +- [ ] T012 [P] [US1] Create [Entity1] model in src/models/[entity1].py +- [ ] T013 [P] [US1] Create [Entity2] model in src/models/[entity2].py +- [ ] T014 [US1] Implement [Service] in src/services/[service].py (depends on T012, T013) +- [ ] T015 [US1] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T016 [US1] Add validation and error handling +- [ ] T017 [US1] Add logging for user story 1 operations + +**Checkpoint**: At this point, User Story 1 should be fully functional and testable independently + +--- + +## Phase 4: User Story 2 - [Title] (Priority: P2) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 2 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T018 [P] [US2] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T019 [P] [US2] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 2 + +- [ ] T020 [P] [US2] Create [Entity] model in src/models/[entity].py +- [ ] T021 [US2] Implement [Service] in src/services/[service].py +- [ ] T022 [US2] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T023 [US2] Integrate with User Story 1 components (if needed) + +**Checkpoint**: At this point, User Stories 1 AND 2 should both work independently + +--- + +## Phase 5: User Story 3 - [Title] (Priority: P3) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 3 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T024 [P] [US3] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T025 [P] [US3] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 3 + +- [ ] T026 [P] [US3] Create [Entity] model in src/models/[entity].py +- [ ] T027 [US3] Implement [Service] in src/services/[service].py +- [ ] T028 [US3] Implement [endpoint/feature] in src/[location]/[file].py + +**Checkpoint**: All user stories should now be independently functional + +--- + +[Add more user story phases as needed, following the same pattern] + +--- + +## Phase N: Polish & Cross-Cutting Concerns + +**Purpose**: Improvements that affect multiple user stories + +- [ ] TXXX [P] Documentation updates in docs/ +- [ ] TXXX Code cleanup and refactoring +- [ ] TXXX Performance optimization across all stories +- [ ] TXXX [P] Additional unit tests (if requested) in tests/unit/ +- [ ] TXXX Security hardening +- [ ] TXXX Run quickstart.md validation + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately +- **Foundational (Phase 2)**: Depends on Setup completion - BLOCKS all user stories +- **User Stories (Phase 3+)**: All depend on Foundational phase completion + - User stories can then proceed in parallel (if staffed) + - Or sequentially in priority order (P1 → P2 → P3) +- **Polish (Final Phase)**: Depends on all desired user stories being complete + +### User Story Dependencies + +- **User Story 1 (P1)**: Can start after Foundational (Phase 2) - No dependencies on other stories +- **User Story 2 (P2)**: Can start after Foundational (Phase 2) - May integrate with US1 but should be independently testable +- **User Story 3 (P3)**: Can start after Foundational (Phase 2) - May integrate with US1/US2 but should be independently testable + +### Within Each User Story + +- Tests (if included) MUST be written and FAIL before implementation +- Models before services +- Services before endpoints +- Core implementation before integration +- Story complete before moving to next priority + +### Parallel Opportunities + +- All Setup tasks marked [P] can run in parallel +- All Foundational tasks marked [P] can run in parallel (within Phase 2) +- Once Foundational phase completes, all user stories can start in parallel (if team capacity allows) +- All tests for a user story marked [P] can run in parallel +- Models within a story marked [P] can run in parallel +- Different user stories can be worked on in parallel by different team members + +--- + +## Parallel Example: User Story 1 + +```bash +# Launch all tests for User Story 1 together (if tests requested): +Task: "Contract test for [endpoint] in tests/contract/test_[name].py" +Task: "Integration test for [user journey] in tests/integration/test_[name].py" + +# Launch all models for User Story 1 together: +Task: "Create [Entity1] model in src/models/[entity1].py" +Task: "Create [Entity2] model in src/models/[entity2].py" +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational (CRITICAL - blocks all stories) +3. Complete Phase 3: User Story 1 +4. **STOP and VALIDATE**: Test User Story 1 independently +5. Deploy/demo if ready + +### Incremental Delivery + +1. Complete Setup + Foundational → Foundation ready +2. Add User Story 1 → Test independently → Deploy/Demo (MVP!) +3. Add User Story 2 → Test independently → Deploy/Demo +4. Add User Story 3 → Test independently → Deploy/Demo +5. Each story adds value without breaking previous stories + +### Parallel Team Strategy + +With multiple developers: + +1. Team completes Setup + Foundational together +2. Once Foundational is done: + - Developer A: User Story 1 + - Developer B: User Story 2 + - Developer C: User Story 3 +3. Stories complete and integrate independently + +--- + +## Notes + +- [P] tasks = different files, no dependencies +- [Story] label maps task to specific user story for traceability +- Each user story should be independently completable and testable +- Verify tests fail before implementing +- Commit after each task or logical group +- Stop at any checkpoint to validate story independently +- Avoid: vague tasks, same file conflicts, cross-story dependencies that break independence diff --git a/.specify/workflows/speckit/workflow.yml b/.specify/workflows/speckit/workflow.yml new file mode 100644 index 00000000..571318af --- /dev/null +++ b/.specify/workflows/speckit/workflow.yml @@ -0,0 +1,74 @@ +schema_version: "1.0" +workflow: + id: "speckit" + name: "Full SDD Cycle" + version: "1.0.1" + author: "GitHub" + description: "Runs specify → plan → tasks → implement with review gates" + +requires: + # 0.8.5 is the first release with engine-side resolution of the + # ``integration: "auto"`` default. Older versions would treat "auto" + # as a literal integration key and fail at dispatch. + speckit_version: ">=0.8.5" + integrations: + # The four commands below (specify, plan, tasks, implement) are core + # spec-kit commands provided by every integration. The list here is an + # advisory, non-exhaustive compatibility hint following the documented + # ``any: [...]`` schema -- it is NOT a closed set. The workflow runs + # against any integration the project was initialized with, including + # ones not listed below, as long as that integration provides the four + # core commands referenced in ``steps``. + any: + - "alquimia" + - "claude" + - "copilot" + - "gemini" + - "opencode" + +inputs: + spec: + type: string + required: true + prompt: "Describe what you want to build" + integration: + type: string + default: "auto" + prompt: "Integration to use (e.g. claude, copilot, gemini; 'auto' uses the project's initialized integration)" + +steps: + - id: specify + command: speckit.specify + integration: "{{ inputs.integration }}" + input: + args: "{{ inputs.spec }}" + + - id: review-spec + type: gate + message: "Review the generated spec before planning." + options: [approve, reject] + on_reject: abort + + - id: plan + command: speckit.plan + integration: "{{ inputs.integration }}" + input: + args: "{{ inputs.spec }}" + + - id: review-plan + type: gate + message: "Review the plan before generating tasks." + options: [approve, reject] + on_reject: abort + + - id: tasks + command: speckit.tasks + integration: "{{ inputs.integration }}" + input: + args: "{{ inputs.spec }}" + + - id: implement + command: speckit.implement + integration: "{{ inputs.integration }}" + input: + args: "{{ inputs.spec }}" diff --git a/.specify/workflows/workflow-registry.json b/.specify/workflows/workflow-registry.json new file mode 100644 index 00000000..9771afb9 --- /dev/null +++ b/.specify/workflows/workflow-registry.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "workflows": { + "speckit": { + "name": "Full SDD Cycle", + "version": "1.0.1", + "description": "Runs specify \u2192 plan \u2192 tasks \u2192 implement with review gates", + "source": "bundled", + "installed_at": "2026-09-24T18:54:51.244156+00:00", + "updated_at": "2026-09-24T18:54:51.244183+00:00" + } + } +} \ No newline at end of file diff --git a/Makefile b/Makefile index c7699462..cae94fd2 100644 --- a/Makefile +++ b/Makefile @@ -112,9 +112,12 @@ _compile_requirements_community_manager: pip-compile --no-emit-index-url backend/core/pyproject.toml backend/community_manager/pyproject.toml --output-file backend/community_manager/requirements.txt -compile_requirements: _compile_requirements_api _compile_requirements_core _compile_requirements_test _compile_requirements_community_manager +_compile_requirements_indexer_gifts: + pip-compile --no-emit-index-url backend/core/pyproject.toml backend/indexer_gifts/pyproject.toml --output-file backend/indexer_gifts/requirements.txt + + +compile_requirements: _compile_requirements_api _compile_requirements_core _compile_requirements_test _compile_requirements_community_manager _compile_requirements_indexer_gifts cp backend/core/requirements.txt backend/indexer_blockchain/requirements.txt - cp backend/core/requirements.txt backend/indexer_gifts/requirements.txt cp backend/core/requirements.txt backend/indexer_price/requirements.txt cp backend/core/requirements.txt backend/indexer_stickers/requirements.txt diff --git a/backend/api/pos/chat.py b/backend/api/pos/chat.py index 863d6e4d..56af5edd 100644 --- a/backend/api/pos/chat.py +++ b/backend/api/pos/chat.py @@ -254,15 +254,15 @@ def validate_category_or_collection(self) -> Self: class TelegramChatGiftRuleCPO(BaseTelegramChatQuantityRuleCPO): - collection_slug: str | None + collection_id: int | None model: str | None = None backdrop: str | None = None pattern: str | None = None @model_validator(mode="after") def validate_category_or_collection(self) -> Self: - if not self.category and not self.collection_slug: - raise ValueError("At least category of collection must be specified") + if not self.category and not self.collection_id: + raise ValueError("At least category or collection must be specified") return self diff --git a/backend/api/pos/gift.py b/backend/api/pos/gift.py index d7582401..0a268ad6 100644 --- a/backend/api/pos/gift.py +++ b/backend/api/pos/gift.py @@ -40,7 +40,8 @@ def from_dto(cls, dto: GiftCollectionsMetadataDTO) -> Self: class GiftFilterPO(GiftFilterDTO): @classmethod def from_query_string(cls, value: str) -> Self: - return cls.model_validate_json(unquote(value)) + data = cls.model_validate_json(unquote(value)) + return data class GiftUniqueInfoFDO(BaseFDO): diff --git a/backend/api/pos/user.py b/backend/api/pos/user.py index 6309553b..ee040f97 100644 --- a/backend/api/pos/user.py +++ b/backend/api/pos/user.py @@ -41,3 +41,9 @@ class UpdateUserWalletFDO(BaseFDO): ), ), ] + + +class RefreshUserGiftsFDO(BaseFDO): + status: str + task_id: str | None = None + message: str diff --git a/backend/api/routes/admin/chat/manage.py b/backend/api/routes/admin/chat/manage.py index ccaf5564..f7093586 100644 --- a/backend/api/routes/admin/chat/manage.py +++ b/backend/api/routes/admin/chat/manage.py @@ -97,11 +97,26 @@ async def update_chat_full_control( requestor=request.state.user, chat_slug=slug, ) - chat = await telegram_chat_action.set_control_level( + chat_result = await telegram_chat_action.set_control_level( is_fully_managed=chat.is_enabled, effective_in_days=chat.effective_in_days, ) - return TelegramChatFDO.model_validate(chat.model_dump()) + return TelegramChatFDO.model_validate(chat_result.model_dump()) + + +@admin_chat_manage_router.post("/control-dry-run") +async def trigger_chat_full_control_dry_run( + request: Request, + slug: str, + db_session: Session = Depends(get_db_session), +) -> TelegramChatFDO: + telegram_chat_action = TelegramChatManageAction( + db_session=db_session, + requestor=request.state.user, + chat_slug=slug, + ) + chat_result = await telegram_chat_action.trigger_control_level_dry_run() + return TelegramChatFDO.model_validate(chat_result.model_dump()) @admin_chat_manage_router.post( diff --git a/backend/api/routes/admin/chat/rule/gift.py b/backend/api/routes/admin/chat/rule/gift.py index f2125b15..f7391e5f 100644 --- a/backend/api/routes/admin/chat/rule/gift.py +++ b/backend/api/routes/admin/chat/rule/gift.py @@ -39,7 +39,7 @@ async def add_chat_gift_rule( ) new_rule = await action.create( group_id=rule.group_id, - collection_slug=rule.collection_slug, + collection_id=rule.collection_id, model=rule.model, backdrop=rule.backdrop, pattern=rule.pattern, @@ -64,7 +64,7 @@ async def update_chat_gift_rule( ) updated_rule = await action.update( rule_id=rule_id, - collection_slug=rule.collection_slug, + collection_id=rule.collection_id, model=rule.model, backdrop=rule.backdrop, pattern=rule.pattern, diff --git a/backend/api/routes/gift.py b/backend/api/routes/gift.py index 79ecf44c..98c2cd82 100644 --- a/backend/api/routes/gift.py +++ b/backend/api/routes/gift.py @@ -47,7 +47,7 @@ async def get_gifts_owners( @gift_router.get( - "/{collection_slug}", + "/{collection_id}", description="Returns an object of gift with their holders (telegram ID and blockchain address)", responses={ HTTP_200_OK: {"model": GiftUniqueItemsFDO}, @@ -58,11 +58,11 @@ async def get_gifts_owners( }, ) async def get_collection_holders( - collection_slug: str, + collection_id: int, db_session: Session = Depends(get_db_session), ) -> GiftUniqueItemsFDO: gift_unique_action = GiftUniqueAction(db_session=db_session) - items = gift_unique_action.get_all(collection_slug=collection_slug) + items = gift_unique_action.get_all(collection_id=collection_id) return GiftUniqueItemsFDO( items=[GiftUniqueInfoFDO.from_dto(item) for item in items] ) diff --git a/backend/api/routes/user.py b/backend/api/routes/user.py index 89d50a58..342b9894 100644 --- a/backend/api/routes/user.py +++ b/backend/api/routes/user.py @@ -16,6 +16,9 @@ ProofValidationError, UserWalletNotConnectedError, ) +from api.pos.user import RefreshUserGiftsFDO +from core.actions.user import UserAction + user_router = APIRouter(prefix="/users") @@ -27,6 +30,31 @@ async def get_user_data( return UserFDO.from_orm(request.state.user) +@user_router.post( + "/me/gifts/refresh", + name="Refresh current user's gifts", + tags=["User", "Gift"], +) +async def refresh_user_gifts( + request: Request, + db_session: Session = Depends(get_db_session), +) -> RefreshUserGiftsFDO: + user_action = UserAction(db_session) + task_id = user_action.refresh_gifts(request.state.user) + + if task_id is None: + raise HTTPException( + status_code=429, + detail="Gifts were recently indexed. Please wait 5 minutes.", + ) + + return RefreshUserGiftsFDO( + status="pending", + task_id=task_id, + message="Gift indexing started.", + ) + + @user_router.post( "/wallet", name="Connect user wallet to the chat with TON Proof", diff --git a/backend/community_manager/actions/chat.py b/backend/community_manager/actions/chat.py index 7265687b..6309427f 100644 --- a/backend/community_manager/actions/chat.py +++ b/backend/community_manager/actions/chat.py @@ -1304,9 +1304,36 @@ async def kick_ineligible_chat_members( :raises MissingChatEntityError: Raised when the chat entity is missing for a member. :raises MissingUserEntityError: Raised when the user entity is missing for a member. """ - ineligible_members = self.authorization_action.get_ineligible_chat_members( - chat_members=chat_members + chat_members_to_check = [ + chat_member + for chat_member in chat_members + if (chat_member.chat.is_full_control or chat_member.is_managed) + and not chat_member.is_admin + ] + + if not chat_members_to_check: + logger.info("No chat members require eligibility check in this chunk") + return + + evaluation_results = ( + self.authorization_action.evaluate_chat_members_eligibility( + chat_members=chat_members_to_check + ) ) + + ineligible_members = [] + for res in evaluation_results: + if not res.is_eligible: + ineligible_members.append(res.member) + elif not res.member.is_managed and res.member.chat.is_full_control: + # User passed the check, but was previously unmanaged in a full control chat. + # Bring them under management so that they are continuously monitored. + res.member.is_managed = True + self.db_session.add(res.member) + logger.debug( + f"User {res.member.user.telegram_id!r} is now managed in chat {res.member.chat_id!r}." + ) + if not ineligible_members: logger.info("No ineligible chat members found") return @@ -1316,3 +1343,75 @@ async def kick_ineligible_chat_members( for member in ineligible_members: # kick_chat_member handles exceptions internally await self.kick_chat_member(member) + + async def check_chat_members_compliance_dry_run(self, chat_id: int) -> None: + """ + Iterates over all members of a chat in batches and performs a dry-run eligibility check. + Logs compliance summaries for ineligible members without kicking them. + + :param chat_id: The ID of the chat to check. + """ + logger.info(f"Starting dry-run check of chat members for chat {chat_id=!r}.") + + total_processed = 0 + total_non_managed = 0 + ineligible_managed_count = 0 + ineligible_non_managed_count = 0 + + for chat_members_chunk in self.telegram_chat_user_service.yield_all_for_chat( + chat_id=chat_id, + batch_size=100, + ): + chat_members_to_check = [ + chat_member + for chat_member in chat_members_chunk + if not chat_member.is_admin + ] + + if not chat_members_to_check: + logger.info( + "No chat members require eligibility check in this chunk. Continue..." + ) + continue + + evaluation_results = ( + self.authorization_action.evaluate_chat_members_eligibility( + chat_members=chat_members_to_check + ) + ) + + for result in evaluation_results: + member_is_managed = result.member.is_managed + + if not member_is_managed: + total_non_managed += 1 + + if not result.is_eligible: + if member_is_managed: + ineligible_managed_count += 1 + else: + ineligible_non_managed_count += 1 + + summary_json = ( + result.summary.model_dump_json() + if result.summary is not None + else "{}" + ) + logger.info( + f"Dry-run: User {result.member.user.telegram_id!r} is ineligible for chat {chat_id!r}. " + f"Managed: {member_is_managed}. Compliance summary: {summary_json}" + ) + + total_processed += len(chat_members_chunk) + logger.info( + f"Dry-run: Processed chunk of {len(chat_members_chunk)} users for chat {chat_id=!r}. " + f"Total processed: {total_processed}" + ) + + logger.info( + f"Dry-run summary for chat {chat_id}: " + f"Total processed: {total_processed}, " + f"Non-managed: {total_non_managed}, " + f"Ineligible (managed): {ineligible_managed_count}, " + f"Ineligible (non-managed): {ineligible_non_managed_count}" + ) diff --git a/backend/community_manager/tasks/chat.py b/backend/community_manager/tasks/chat.py index 7d7390e8..f889fe7f 100644 --- a/backend/community_manager/tasks/chat.py +++ b/backend/community_manager/tasks/chat.py @@ -56,6 +56,21 @@ def check_target_chat_members_task(chat_id: int) -> None: async_to_sync(check_target_chat_members)(chat_id) +async def check_target_chat_members_dry_run(chat_id: int) -> None: + with DBService().db_session() as db_session: + action = CommunityManagerUserChatAction(db_session) + await action.check_chat_members_compliance_dry_run(chat_id=chat_id) + + +@app.task( + name="check-target-chat-members-dry-run", + queue=CELERY_SYSTEM_QUEUE_NAME, + ignore_result=True, +) +def check_target_chat_members_dry_run_task(chat_id: int) -> None: + async_to_sync(check_target_chat_members_dry_run)(chat_id) + + async def refresh_chat_external_sources_async() -> None: with DBService().db_session() as db_session: # BotAPI does not need a telethon client diff --git a/backend/core/src/core/actions/authorization.py b/backend/core/src/core/actions/authorization.py index 55ccf06e..760692eb 100644 --- a/backend/core/src/core/actions/authorization.py +++ b/backend/core/src/core/actions/authorization.py @@ -16,6 +16,7 @@ EligibilitySummaryStickerCollectionInternalDTO, EligibilitySummaryJettonInternalDTO, EligibilitySummaryNftCollectionInternalDTO, + ChatMemberEligibilityResultDTO, ) from core.dtos.gift.collection import GiftCollectionDTO from core.dtos.resource import JettonDTO, NftCollectionDTO @@ -220,34 +221,23 @@ def get_eligibility_rules( emoji=all_emoji_rules, ) - def get_ineligible_chat_members( + def evaluate_chat_members_eligibility( self, chat_members: list[TelegramChatUser], - ) -> list[TelegramChatUser]: + ) -> list[ChatMemberEligibilityResultDTO]: """ - Determines and returns a list of chat members who are ineligible to be part of their respective chats based on + Determines and returns eligibility evaluations for a list of chat members based on eligibility rules and various related data sources such as wallets, NFTs, and gifts. This functionality checks each member's eligibility against the chat's specific rules, wallet information, NFTs, jettons, stickers, and - other items associated with the user. Users that do not meet the criteria defined by the chat's eligibility - rules are considered ineligible. + other items associated with the user. :param chat_members: A list of TelegramChatUser objects representing the members of various Telegram chats. - :return: A list of TelegramChatUser objects representing chat members who are not eligible to be part of their - respective chats. + :return: A list of ChatMemberEligibilityResultDTO objects representing the eligibility of each member. """ members_per_chat = defaultdict(list) user_id_to_telegram_id = {} eligibility_rules_per_chat: dict[int, TelegramChatEligibilityRulesDTO] = {} - chat_members = [ - # Skip checks for non-managed users in the chats where full control is disabled - # and skip checks for admins - chat_member - for chat_member in chat_members - if (chat_member.chat.is_full_control or chat_member.is_managed) - and not chat_member.is_admin - ] - if not chat_members: logger.info("No chat members to check eligibility for. Skipping.") return [] @@ -295,38 +285,39 @@ def get_ineligible_chat_members( telegram_user_id=user_id_to_telegram_id[user_id] ) - ineligible_members = [] + results = [] for chat, members in members_per_chat.items(): for member in members: member_wallet = ( member.wallet_link.wallet if member.wallet_link else None ) member_wallet_address = member_wallet.address if member_wallet else None - if not ( - eligibility_summary := self.check_chat_member_eligibility( - eligibility_rules=eligibility_rules_per_chat[chat], - user=member.user, - user_wallet=member_wallet, - user_jettons=jetton_wallets_per_wallet.get( - member_wallet_address, [] - ), - user_nft_items=nft_items_per_wallet.get( - member_wallet_address, [] - ), - user_sticker_items=sticker_items_per_user.get( - member.user_id, [] - ), - user_gift_items=gift_items_per_user.get(member.user_id, []), - chat_member=member, - ) - ): + eligibility_summary = self.check_chat_member_eligibility( + eligibility_rules=eligibility_rules_per_chat[chat], + user=member.user, + user_wallet=member_wallet, + user_jettons=jetton_wallets_per_wallet.get( + member_wallet_address, [] + ), + user_nft_items=nft_items_per_wallet.get(member_wallet_address, []), + user_sticker_items=sticker_items_per_user.get(member.user_id, []), + user_gift_items=gift_items_per_user.get(member.user_id, []), + chat_member=member, + ) + if not eligibility_summary: logger.debug( f"User {member.user.telegram_id!r} is not eligible to be in chat {chat!r}." f"Eligibility summary: {eligibility_summary!r}" ) - ineligible_members.append(member) + results.append( + ChatMemberEligibilityResultDTO( + member=member, + is_eligible=bool(eligibility_summary), + summary=eligibility_summary, + ) + ) - return ineligible_members + return results @classmethod def check_chat_member_eligibility( @@ -378,7 +369,7 @@ def check_chat_member_eligibility( type=EligibilityCheckType.TONCOIN, category=rule.category, expected=rule.threshold, - title="TON", + title="GRAM", actual=(user_wallet.balance if user_wallet else None) or 0, is_enabled=rule.is_enabled, ) diff --git a/backend/core/src/core/actions/chat/__init__.py b/backend/core/src/core/actions/chat/__init__.py index c2d4915c..edd7cf6c 100644 --- a/backend/core/src/core/actions/chat/__init__.py +++ b/backend/core/src/core/actions/chat/__init__.py @@ -371,6 +371,35 @@ async def refresh_participants(self) -> None: redis_service.delete(f"refresh_chat_{self.chat.id}") raise + async def trigger_control_level_dry_run(self) -> TelegramChatDTO: + """ + Triggers a dry-run of the chat control level checks. + Uses a 5-minute rate limit to prevent spamming. + """ + redis_service = RedisService() + if not redis_service.set( + f"set_control_level_dry_run_{self.chat.id}", "1", ex=300, nx=True + ): + logger.warning( + "An attempt to spam trigger_control_level_dry_run in chat %d", + self.chat.id, + ) + raise HTTPException( + status_code=HTTP_429_TOO_MANY_REQUESTS, + detail="Too many requests. Please wait 5 minutes before trying again.", + ) + + logger.info(f"Triggering control level dry run for chat {self.chat.id}") + + # Enqueue the dry run task + sender.send_task( + "check-target-chat-members-dry-run", + args=(self.chat.id,), + queue=CELERY_SYSTEM_QUEUE_NAME, + ) + + return TelegramChatDTO.from_object(self.chat) + async def get_with_eligibility_rules(self) -> TelegramChatWithRulesDTO: """ This is an administrative method to get chat with rules that includes disabled rules diff --git a/backend/core/src/core/actions/chat/rule/blockchain.py b/backend/core/src/core/actions/chat/rule/blockchain.py index fcf958fe..20af3738 100644 --- a/backend/core/src/core/actions/chat/rule/blockchain.py +++ b/backend/core/src/core/actions/chat/rule/blockchain.py @@ -551,11 +551,11 @@ def create( logs the creation event and returns a data transfer object (DTO) that represents the newly created TON rule. - :param category: The currency category to associate with the new TON rule. - :param threshold: The minimum threshold value for the TON rule. - :param group_id: The group ID to associate with the new TON rule, if applicable. + :param category: The currency category to associate with the new GRAM rule. + :param threshold: The minimum threshold value for the GRAM rule. + :param group_id: The group ID to associate with the new GRAM rule, if applicable. :return: A DTO representing the chat eligibility rule created from the - TON rule. + GRAM rule. :raises HTTPException: If a duplicate rule of the specified type and category is found. """ group_id = self.resolve_group_id(group_id=group_id) @@ -569,7 +569,7 @@ def create( is_enabled=True, ) ) - logger.info(f"Chat {self.chat.id!r} linked to a new TON rule") + logger.info(f"Chat {self.chat.id!r} linked to a new GRAM rule") self.refresh_chat_floor_price() return ChatEligibilityRuleDTO.from_toncoin_rule(new_rule) @@ -581,7 +581,7 @@ def update( is_enabled: bool, ) -> ChatEligibilityRuleDTO: """ - Updates an existing chat eligibility rule for TON based on the provided attributes. + Updates an existing chat eligibility rule for GRAM based on the provided attributes. Retrieves the rule by its ID and ensures it belongs to the current chat. Checks for duplicate rules before applying updates. @@ -589,7 +589,7 @@ def update( :param category: The category of the currency for the rule, or None if not applicable. :param threshold: The threshold value associated with the rule, could be an integer or float. :param is_enabled: A boolean indicating whether the rule is enabled or disabled. - :return: A data transfer object (DTO) representing the updated chat eligibility rule for TON. + :return: A data transfer object (DTO) representing the updated chat eligibility rule for GRAM. :raises HTTPException: If the rule with the provided ID does not exist, or if a duplicate rule of the specified type and category is found. """ @@ -615,7 +615,7 @@ def update( is_enabled=is_enabled, ), ) - logger.info(f"Updated chat jetton rule {rule_id!r} for TON") + logger.info(f"Updated chat jetton rule {rule_id!r} for GRAM") self.refresh_chat_floor_price() return ChatEligibilityRuleDTO.from_toncoin_rule(updated_rule) @@ -630,6 +630,6 @@ def delete(self, rule_id: int) -> None: status_code=HTTP_404_NOT_FOUND, ) self.telegram_chat_toncoin_service.delete(rule_id, chat_id=self.chat.id) - logger.info(f"Deleted chat TON rule {rule_id!r}") + logger.info(f"Deleted chat GRAM rule {rule_id!r}") self.refresh_chat_floor_price() self.remove_group_if_empty(group_id=group_id) diff --git a/backend/core/src/core/actions/chat/rule/gift.py b/backend/core/src/core/actions/chat/rule/gift.py index b193a887..82c7d160 100644 --- a/backend/core/src/core/actions/chat/rule/gift.py +++ b/backend/core/src/core/actions/chat/rule/gift.py @@ -14,6 +14,7 @@ from core.models.user import User from core.services.chat.rule.gift import TelegramChatGiftCollectionService from core.services.gift.item import GiftUniqueService +from core.services.gift.collection import GiftCollectionService logger = logging.getLogger(__name__) @@ -23,6 +24,7 @@ def __init__(self, db_session: Session, requestor: User, chat_slug: str): super().__init__(db_session, requestor, chat_slug) self.service = TelegramChatGiftCollectionService(db_session) self.gift_unique_service = GiftUniqueService(db_session) + self.gift_collection_service = GiftCollectionService(db_session) async def read(self, rule_id: int) -> GiftChatEligibilityRuleDTO: try: @@ -36,7 +38,7 @@ def check_duplicates( self, chat_id: int, group_id: int, - collection_slug: str | None, + collection_id: int | None, category: str | None, entity_id: int | None = None, ) -> None: @@ -47,7 +49,7 @@ def check_duplicates( :param chat_id: The unique identifier for the chat where the rule applies. :param group_id: The unique identifier for the group where the rule applies. - :param collection_slug: The slug identifying the collection; can be None if not applicable. + :param collection_id: The id identifying the collection; can be None if not applicable. :param category: The category to which the rule applies; can be None if not applicable. :param entity_id: Optional identifier for the specific entity to exclude from duplicate checks. @@ -56,7 +58,7 @@ def check_duplicates( existing_rules = self.service.find( chat_id=chat_id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, ) if next(filter(lambda rule: rule.id != entity_id, existing_rules), None): @@ -67,41 +69,40 @@ def check_duplicates( def validate_params( self, - collection_slug: str | None, + collection_id: int | None, model: str | None, backdrop: str | None, pattern: str | None, ) -> None: - # If the collection slug is not set or attributes are not selected – no need to validate them - if not collection_slug or not any((model, backdrop, pattern)): + # If the collection id is not set or attributes are not selected – no need to validate them + if not collection_id or not any((model, backdrop, pattern)): return - options = self.gift_unique_service.get_unique_options( - collection_slug=collection_slug - ) + collection = self.gift_collection_service.get(collection_id) + options = collection.options if model and model not in options.get("models", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Model {model!r} is not available for the collection {collection_slug!r}.", + detail=f"Model {model!r} is not available for the collection {collection_id!r}.", ) if backdrop and backdrop not in options.get("backdrops", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Backdrop {backdrop!r} is not available for the collection {collection_slug!r}.", + detail=f"Backdrop {backdrop!r} is not available for the collection {collection_id!r}.", ) if pattern and pattern not in options.get("patterns", []): raise HTTPException( status_code=HTTP_400_BAD_REQUEST, - detail=f"Pattern {pattern!r} is not available for the collection {collection_slug!r}.", + detail=f"Pattern {pattern!r} is not available for the collection {collection_id!r}.", ) async def create( self, group_id: int | None, - collection_slug: str | None, + collection_id: int | None, model: str | None, backdrop: str | None, pattern: str | None, @@ -113,16 +114,16 @@ async def create( self.check_duplicates( chat_id=self.chat.id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, ) - self.validate_params(collection_slug, model, backdrop, pattern) + self.validate_params(collection_id, model, backdrop, pattern) new_rule = self.service.create( CreateTelegramChatGiftCollectionRuleDTO( chat_id=self.chat.id, group_id=group_id, - collection_slug=collection_slug, + collection_id=collection_id, model=model, backdrop=backdrop, pattern=pattern, @@ -139,7 +140,7 @@ async def create( async def update( self, rule_id: int, - collection_slug: str | None, + collection_id: int | None, category: str | None, model: str | None, backdrop: str | None, @@ -155,16 +156,16 @@ async def update( self.check_duplicates( chat_id=self.chat.id, group_id=rule.group_id, - collection_slug=collection_slug, + collection_id=collection_id, category=category, entity_id=rule_id, ) - self.validate_params(collection_slug, model, backdrop, pattern) + self.validate_params(collection_id, model, backdrop, pattern) updated_rule = self.service.update( rule=rule, dto=UpdateTelegramChatGiftCollectionRuleDTO( - collection_slug=collection_slug, + collection_id=collection_id, category=category, threshold=threshold, is_enabled=is_enabled, diff --git a/backend/core/src/core/actions/gift.py b/backend/core/src/core/actions/gift.py index 5c1945c0..d93a5a3f 100644 --- a/backend/core/src/core/actions/gift.py +++ b/backend/core/src/core/actions/gift.py @@ -10,6 +10,7 @@ from core.constants import GIFT_COLLECTIONS_METADATA_KEY from core.dtos.gift.collection import ( GiftCollectionMetadataDTO, + GiftCollectionOptionsDTO, GiftCollectionsMetadataDTO, GiftFilterDTO, GiftFiltersDTO, @@ -19,7 +20,6 @@ from core.services.gift.collection import GiftCollectionService from core.services.gift.item import GiftUniqueService from core.services.superredis import RedisService -from core.settings import core_settings from core.utils.cache import cached_dto_result @@ -33,30 +33,29 @@ def __init__(self, db_session: Session) -> None: @cached_dto_result( cache_key=GIFT_COLLECTIONS_METADATA_KEY, response_model=GiftCollectionsMetadataDTO, - cache_ttl=60 * 60 * 24, # 1-day cache + cache_ttl=60 * 5, # 5-minute cache ) def get_metadata(self) -> GiftCollectionsMetadataDTO: - all_collections = self.collection_service.get_all( - slugs=core_settings.whitelisted_gift_collections - ) + all_collections = self.collection_service.get_all() collections_with_options = [] for collection in all_collections: - options = self.service.get_unique_options(collection_slug=collection.slug) + options = collection.options collections_with_options.append( GiftCollectionMetadataDTO( - slug=collection.slug, + id=collection.id, title=collection.title, preview_url=collection.preview_url, supply=collection.supply, upgraded_count=collection.upgraded_count, - models=options["models"], - backdrops=options["backdrops"], - patterns=options["patterns"], + options=GiftCollectionOptionsDTO( + models=options["models"], + backdrops=options["backdrops"], + patterns=options["patterns"], + ), ) ) - return GiftCollectionsMetadataDTO(collections=collections_with_options) @staticmethod @@ -89,7 +88,7 @@ def __construct_filter_options_query( for option in options: # Basic filtering logic (collection, model, backdrop, pattern) base_filter = and_( - GiftUnique.collection_slug == option.collection, + GiftUnique.collection_id == option.collection_id, GiftUnique.telegram_owner_id.isnot(None), *filter( None.__ne__, @@ -144,18 +143,18 @@ def get_collections_holders(self, options: list[GiftFilterDTO]) -> Sequence[int] result = self.db_session.execute(query).scalars().all() return result - def get_all(self, collection_slug: str) -> Sequence[GiftUniqueDTO]: + def get_all(self, collection_id: int) -> Sequence[GiftUniqueDTO]: """ Fetches all unique items in a given collection. """ try: - self.collection_service.get(slug=collection_slug) + self.collection_service.get(id=collection_id) except NoResultFound: raise HTTPException( status_code=HTTP_404_NOT_FOUND, - detail=f"Collection {collection_slug!r} not found", + detail=f"Collection {collection_id!r} not found", ) return [ GiftUniqueDTO.from_orm(gift) - for gift in self.service.get_all(collection_slug=collection_slug) + for gift in self.service.get_all(collection_id=collection_id) ] diff --git a/backend/core/src/core/actions/user.py b/backend/core/src/core/actions/user.py index c9e6083e..2df63428 100644 --- a/backend/core/src/core/actions/user.py +++ b/backend/core/src/core/actions/user.py @@ -6,7 +6,12 @@ from core.actions.base import BaseAction from core.dtos.user import TelegramUserDTO from core.models.user import User - +from core.services.superredis import RedisService +from core.constants import ( + CELERY_GIFT_USER_PRIORITY_QUEUE, + USER_GIFT_REFRESH_COOLDOWN_SECONDS, +) +from core.utils.task import sender logger = logging.getLogger(__name__) @@ -15,65 +20,53 @@ class UserAction(BaseAction): def __init__(self, db_session: Session) -> None: super().__init__(db_session) + def refresh_gifts(self, user: User) -> str | None: + """ + Triggers user gift indexing if not rate-limited. + Returns task_id if dispatched, None if rate-limited. + """ + redis_service = RedisService() + rate_limit_key = f"user-gift-refresh-{user.telegram_id}" + if not redis_service.set( + rate_limit_key, "1", ex=USER_GIFT_REFRESH_COOLDOWN_SECONDS, nx=True + ): + logger.info(f"User {user.telegram_id} gift refresh rate-limited.") + return None + + task_result = sender.send_task( + "index-user-gifts", + args=(user.telegram_id,), + queue=CELERY_GIFT_USER_PRIORITY_QUEUE, + ) + logger.info( + f"Dispatched gift indexing task {task_result.id} for user {user.telegram_id}" + ) + return task_result.id + def _initial_user_indexing(self, user: User) -> None: """ Indexes the user's data upon their creation in the system. This method is - intended for internal usage to initialize the indexing process for a user. It logs the - process of indexing and ensures the data related to the user is properly - indexed using internal and external service(-s). - - :param user: The user entity for which the data should be indexed. + intended for internal usage to initialize the indexing process for a user. """ logger.debug(f"Indexing user {user.id!r} upon creation...") + self.refresh_gifts(user) def create(self, telegram_user: TelegramUserDTO) -> User: - """ - Creates a new user in the system based on the provided Telegram user data. - - This method takes in a Telegram user data transfer object (DTO) and creates - a new user. It also performs an initial indexing of the created user. - - :param telegram_user: The DTO containing data of the Telegram user to - be transformed into a system user. - :return: The newly created user object. - """ user = self.user_service.create(telegram_user) self._initial_user_indexing(user) return user def create_or_update(self, telegram_user: TelegramUserDTO) -> User: - """ - Creates or updates a user based on the provided Telegram user data. If a user with the given - Telegram ID exists, their data is updated. Otherwise, a new user is created, and an initial - indexing operation is performed on the new user. - - :param telegram_user: Data Transfer Object containing information about the Telegram user. - :return: The updated or newly created user instance. - """ try: user = self.user_service.get_by_telegram_id(telegram_user.id) user = self.user_service.update(user, telegram_user) except NoResultFound: user = self.create(telegram_user) - return user def get_or_create(self, telegram_user: TelegramUserDTO) -> User: - """ - Retrieves an existing user by their Telegram ID or creates a new user if no such - user exists. This function ensures that the user's data is either fetched - from the database or initialized as a new entry. The retrieval or creation - depends on the presence of a matching Telegram ID. Additionally, indexing - is performed for newly created users to set up required infrastructure. - - :param telegram_user: An object containing Telegram user data necessary - for either fetching or creating a User record. - :return: The user object associated with the provided Telegram user data, either - fetched from the database or newly created. - """ try: user = self.user_service.get_by_telegram_id(telegram_user.id) except NoResultFound: user = self.create(telegram_user) - return user diff --git a/backend/core/src/core/constants.py b/backend/core/src/core/constants.py index f8ef03ce..1aeffa2f 100644 --- a/backend/core/src/core/constants.py +++ b/backend/core/src/core/constants.py @@ -43,8 +43,11 @@ CELERY_GATEWAY_INDEX_QUEUE_NAME = "gateway-index-queue" CELERY_INDEX_PRICES_QUEUE_NAME = "index-prices-queue" # Gifts -GIFT_COLLECTIONS_METADATA_KEY = "gifts-metadata" -CELERY_GIFT_FETCH_QUEUE_NAME = "gift-fetch-queue" +GIFT_COLLECTIONS_METADATA_KEY = "gifts-metadata-v3" +CELERY_GIFT_COLLECTIONS_SYNC_QUEUE = "gift-collections-sync" +CELERY_GIFT_USER_PRIORITY_QUEUE = "gift-user-priority" +USER_GIFT_REFRESH_COOLDOWN_SECONDS = 300 +CELERY_GIFT_USER_BATCH_QUEUE = "gift-user-batch" UPDATED_GIFT_USER_IDS = "updated_gift_user_ids" # ----------------- Paths --------------------- diff --git a/backend/core/src/core/dtos/chat/rule/__init__.py b/backend/core/src/core/dtos/chat/rule/__init__.py index 5cfc4548..66d62b90 100644 --- a/backend/core/src/core/dtos/chat/rule/__init__.py +++ b/backend/core/src/core/dtos/chat/rule/__init__.py @@ -83,7 +83,7 @@ def from_toncoin_rule(cls, rule: TelegramChatToncoin) -> Self: id=rule.id, group_id=rule.group_id, type=EligibilityCheckType.TONCOIN, - title="TON", + title="GRAM", expected=rule.threshold, photo_url="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTYiIGhlaWdodD0iNTYiIHZpZXdCb3g9IjAgMCA1NiA1NiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTI4IDU2QzQzLjQ2NCA1NiA1NiA0My40NjQgNTYgMjhDNTYgMTIuNTM2IDQzLjQ2NCAwIDI4IDBDMTIuNTM2IDAgMCAxMi41MzYgMCAyOEMwIDQzLjQ2NCAxMi41MzYgNTYgMjggNTZaIiBmaWxsPSIjMDA5OEVBIi8+CjxwYXRoIGQ9Ik0zNy41NjAzIDE1LjYyNzdIMTguNDM4NkMxNC45MjI4IDE1LjYyNzcgMTIuNjk0NCAxOS40MjAyIDE0LjQ2MzIgMjIuNDg2MUwyNi4yNjQ0IDQyLjk0MDlDMjcuMDM0NSA0NC4yNzY1IDI4Ljk2NDQgNDQuMjc2NSAyOS43MzQ1IDQyLjk0MDlMNDEuNTM4MSAyMi40ODYxQzQzLjMwNDUgMTkuNDI1MSA0MS4wNzYxIDE1LjYyNzcgMzcuNTYyNyAxNS42Mjc3SDM3LjU2MDNaTTI2LjI1NDggMzYuODA2OEwyMy42ODQ3IDMxLjgzMjdMMTcuNDgzMyAyMC43NDE0QzE3LjA3NDIgMjAuMDMxNSAxNy41Nzk1IDE5LjEyMTggMTguNDM2MiAxOS4xMjE4SDI2LjI1MjRWMzYuODA5MkwyNi4yNTQ4IDM2LjgwNjhaTTM4LjUxMDggMjAuNzM5TDMyLjMxMTggMzEuODM1MUwyOS43NDE3IDM2LjgwNjhWMTkuMTE5NEgzNy41NTc5QzM4LjQxNDYgMTkuMTE5NCAzOC45MTk5IDIwLjAyOTEgMzguNTEwOCAyMC43MzlaIiBmaWxsPSJ3aGl0ZSIvPgo8L3N2Zz4K", blockchain_address=None, diff --git a/backend/core/src/core/dtos/chat/rule/gift.py b/backend/core/src/core/dtos/chat/rule/gift.py index ed843793..0d32a563 100644 --- a/backend/core/src/core/dtos/chat/rule/gift.py +++ b/backend/core/src/core/dtos/chat/rule/gift.py @@ -12,17 +12,17 @@ class BaseTelegramChatGiftCollectionRuleDTO(BaseModel): threshold: int is_enabled: bool - collection_slug: str | None + collection_id: int | None model: str | None backdrop: str | None pattern: str | None category: str | None @model_validator(mode="after") - def validate_slug_or_category(self) -> Self: - if (self.category is None) == (self.collection_slug is None): + def validate_id_or_category(self) -> Self: + if (self.category is None) == (self.collection_id is None): raise ValueError( - "Either category or collection_slug must be provided and not both." + "Either category or collection_id must be provided and not both." ) return self @@ -73,9 +73,8 @@ def promote_url(self) -> str | None: # FIXME: Turn on when market is released # if self.collection: # return PROMOTE_GIFT_COLLECTION_TEMPLATE.format( - # collection_slug=self.collection.slug + # collection_id=self.collection.id # ) - return None @classmethod diff --git a/backend/core/src/core/dtos/chat/rule/internal.py b/backend/core/src/core/dtos/chat/rule/internal.py index 504939f2..bc62b1a5 100644 --- a/backend/core/src/core/dtos/chat/rule/internal.py +++ b/backend/core/src/core/dtos/chat/rule/internal.py @@ -1,10 +1,11 @@ -from pydantic import BaseModel, computed_field +from pydantic import BaseModel, ConfigDict, computed_field from core.enums.rule import EligibilityCheckType from core.dtos.gift.collection import GiftCollectionDTO from core.dtos.resource import NftCollectionDTO, JettonDTO from core.dtos.sticker import MinimalStickerCollectionDTO, MinimalStickerCharacterDTO from core.enums.nft import NftCollectionAsset +from core.models.chat import TelegramChatUser class EligibilitySummaryInternalDTO(BaseModel): @@ -110,3 +111,11 @@ def __bool__(self): def __repr__(self): return f"<{self.__class__.__name__} ({self.items=})>" + + +class ChatMemberEligibilityResultDTO(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + member: TelegramChatUser + is_eligible: bool + summary: RulesEligibilitySummaryInternalDTO | None diff --git a/backend/core/src/core/dtos/fields.py b/backend/core/src/core/dtos/fields.py new file mode 100644 index 00000000..7dad0597 --- /dev/null +++ b/backend/core/src/core/dtos/fields.py @@ -0,0 +1,7 @@ +from typing import Annotated + +from pydantic import PlainSerializer + +StringifiedInt = Annotated[ + int, PlainSerializer(lambda x: str(x), return_type=str, when_used="json") +] diff --git a/backend/core/src/core/dtos/gift/collection.py b/backend/core/src/core/dtos/gift/collection.py index 5d1b5cf9..32b52261 100644 --- a/backend/core/src/core/dtos/gift/collection.py +++ b/backend/core/src/core/dtos/gift/collection.py @@ -4,49 +4,57 @@ from pydantic import BaseModel from telethon.tl.types import StarGiftUnique +from core.dtos.fields import StringifiedInt from core.models.gift import GiftCollection +class GiftCollectionOptionsDTO(BaseModel): + models: list[str] + backdrops: list[str] + patterns: list[str] + + class GiftCollectionDTO(BaseModel): - slug: str + id: StringifiedInt title: str preview_url: str | None supply: int upgraded_count: int + options: GiftCollectionOptionsDTO last_updated: datetime.datetime @classmethod def from_orm(cls, obj: GiftCollection) -> Self: return cls( - slug=obj.slug, + id=obj.id, title=obj.title, preview_url=obj.preview_url, supply=obj.supply, upgraded_count=obj.upgraded_count, + options=obj.options, last_updated=obj.last_updated, ) @classmethod - def from_telethon(cls, slug: str, obj: StarGiftUnique, preview_url: str) -> Self: + def from_telethon(cls, id: int, obj: StarGiftUnique, preview_url: str) -> Self: return cls( - slug=slug, + id=id, title=obj.title, preview_url=preview_url, supply=obj.availability_total, upgraded_count=obj.availability_issued, + options=GiftCollectionOptionsDTO(models=[], backdrops=[], patterns=[]), last_updated=datetime.datetime.now(tz=datetime.UTC), ) class GiftCollectionMetadataDTO(BaseModel): - slug: str + id: StringifiedInt title: str preview_url: str | None supply: int upgraded_count: int - models: list[str] - backdrops: list[str] - patterns: list[str] + options: GiftCollectionOptionsDTO class GiftCollectionsMetadataDTO(BaseModel): @@ -54,7 +62,7 @@ class GiftCollectionsMetadataDTO(BaseModel): class GiftFilterDTO(BaseModel): - collection: str + collection_id: StringifiedInt model: str | None = None backdrop: str | None = None pattern: str | None = None @@ -68,26 +76,31 @@ class GiftFiltersDTO(BaseModel): def validate_with_context( cls, objs: list[GiftFilterDTO], context: GiftCollectionsMetadataDTO ) -> Self: - context_by_slug = { - collection.slug: collection for collection in context.collections + context_by_id = { + collection.id: collection for collection in context.collections } for obj in objs: - if not (collection_metadata := context_by_slug.get(obj.collection)): - raise ValueError(f"Collection {obj.collection} not found in metadata") + if not (collection_metadata := context_by_id.get(obj.collection_id)): + raise ValueError( + f"Collection {obj.collection_id} not found in metadata" + ) - if obj.model and obj.model not in collection_metadata.models: + if obj.model and obj.model not in collection_metadata.options.models: raise ValueError( - f"Model {obj.model} not found in collection {obj.collection}" + f"Model {obj.model} not found in collection {obj.collection_id}" ) - if obj.backdrop and obj.backdrop not in collection_metadata.backdrops: + if ( + obj.backdrop + and obj.backdrop not in collection_metadata.options.backdrops + ): raise ValueError( - f"Backdrop {obj.backdrop} not found in collection {obj.collection}" + f"Backdrop {obj.backdrop} not found in collection {obj.collection_id}" ) - if obj.pattern and obj.pattern not in collection_metadata.patterns: + if obj.pattern and obj.pattern not in collection_metadata.options.patterns: raise ValueError( - f"Pattern {obj.pattern} not found in collection {obj.collection}" + f"Pattern {obj.pattern} not found in collection {obj.collection_id}" ) return cls(filters=objs) diff --git a/backend/core/src/core/dtos/gift/item.py b/backend/core/src/core/dtos/gift/item.py index 95133d87..23ce8750 100644 --- a/backend/core/src/core/dtos/gift/item.py +++ b/backend/core/src/core/dtos/gift/item.py @@ -9,12 +9,13 @@ StarGiftAttributePattern, ) +from core.dtos.fields import StringifiedInt from core.models.gift import GiftUnique class GiftUniqueDTO(BaseModel): slug: str - collection_slug: str + collection_id: StringifiedInt telegram_owner_id: int | None number: int blockchain_address: str | None @@ -28,7 +29,7 @@ class GiftUniqueDTO(BaseModel): def from_orm(cls, obj: GiftUnique) -> Self: return cls( slug=obj.slug, - collection_slug=obj.collection_slug, + collection_id=obj.collection_id, telegram_owner_id=obj.telegram_owner_id, number=obj.number, blockchain_address=obj.blockchain_address, @@ -40,7 +41,7 @@ def from_orm(cls, obj: GiftUnique) -> Self: ) @classmethod - def from_telethon(cls, collection_slug: str, obj: StarGiftUnique) -> Self: + def from_telethon(cls, collection_id: int, obj: StarGiftUnique) -> Self: model_attribute = next( ( attribute @@ -73,7 +74,7 @@ def from_telethon(cls, collection_slug: str, obj: StarGiftUnique) -> Self: return cls( slug=obj.slug, - collection_slug=collection_slug, + collection_id=collection_id, telegram_owner_id=getattr(obj.owner_id, "user_id", None), number=obj.num, blockchain_address=obj.gift_address, diff --git a/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json b/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json new file mode 100644 index 00000000..a81ce01f --- /dev/null +++ b/backend/core/src/core/migrations/data/gift-collection-ids-to-names.json @@ -0,0 +1,111 @@ +{ + "5983471780763796287": "Santa Hat", + "5936085638515261992": "Signet Ring", + "5933671725160989227": "Precious Peach", + "5936013938331222567": "Plush Pepe", + "5913442287462908725": "Spiced Wine", + "5915502858152706668": "Jelly Bunny", + "5915521180483191380": "Durov's Cap", + "5913517067138499193": "Perfume Bottle", + "5882125812596999035": "Eternal Rose", + "5882252952218894938": "Berry Box", + "5857140566201991735": "Vintage Cigar", + "5846226946928673709": "Magic Potion", + "5845776576658015084": "Kissed Frog", + "5825801628657124140": "Hex Pot", + "5825480571261813595": "Evil Eye", + "5841689550203650524": "Sharp Tongue", + "5841391256135008713": "Trapped Heart", + "5839038009193792264": "Skull Flower", + "5837059369300132790": "Scared Cat", + "5821261908354794038": "Spy Agaric", + "5783075783622787539": "Homemade Cake", + "5933531623327795414": "Genie Lamp", + "6028426950047957932": "Lunar Snake", + "6003643167683903930": "Party Sparkler", + "5933590374185435592": "Jester Hat", + "5821384757304362229": "Witch Hat", + "5915733223018594841": "Hanging Star", + "5915550639663874519": "Love Candle", + "6001538689543439169": "Cookie Heart", + "5782988952268964995": "Desk Calendar", + "6001473264306619020": "Jingle Bells", + "5980789805615678057": "Snow Mittens", + "5836780359634649414": "Voodoo Doll", + "5841632504448025405": "Mad Pumpkin", + "5825895989088617224": "Hypno Lollipop", + "5782984811920491178": "B-Day Candle", + "5935936766358847989": "Bunny Muffin", + "5933629604416717361": "Astral Shard", + "5837063436634161765": "Flying Broom", + "5841336413697606412": "Crystal Ball", + "5821205665758053411": "Eternal Candle", + "5936043693864651359": "Swiss Watch", + "5983484377902875708": "Ginger Cookie", + "5879737836550226478": "Mini Oscar", + "5170594532177215681": "Lol Pop", + "5843762284240831056": "Ion Gem", + "5936017773737018241": "Star Notepad", + "5868659926187901653": "Loot Bag", + "5868348541058942091": "Love Potion", + "5868220813026526561": "Toy Bear", + "5868503709637411929": "Diamond Ring", + "5167939598143193218": "Sakura Flower", + "5981026247860290310": "Sleigh Bell", + "5897593557492957738": "Top Hat", + "5856973938650776169": "Record Player", + "5983259145522906006": "Winter Wreath", + "5981132629905245483": "Snow Globe", + "5846192273657692751": "Electric Skull", + "6023752243218481939": "Tama Gadget", + "6003373314888696650": "Candy Cane", + "5933793770951673155": "Neko Helmet", + "6005659564635063386": "Jack-in-the-Box", + "5773668482394620318": "Easter Egg", + "5870661333703197240": "Bonded Ring", + "6023917088358269866": "Pet Snake", + "6023679164349940429": "Snake Box", + "6003767644426076664": "Xmas Stocking", + "6028283532500009446": "Big Year", + "6003735372041814769": "Holiday Drink", + "5859442703032386168": "Gem Signet", + "5897581235231785485": "Light Sword", + "5870784783948186838": "Restless Jar", + "5870720080265871962": "Nail Bracelet", + "5895328365971244193": "Heroic Helmet", + "5895544372761461960": "Bow Tie", + "5868455043362980631": "Heart Locket", + "5871002671934079382": "Lush Bouquet", + "5933543975653737112": "Whip Cupcake", + "5870862540036113469": "Joyful Bundle", + "5868561433997870501": "Cupid Charm", + "5868595669182186720": "Valentine Box", + "6014591077976114307": "Snoop Dogg", + "6012607142387778152": "Swag Bag", + "6012435906336654262": "Snoop Cigar", + "6014675319464657779": "Low Rider", + "6014697240977737490": "Westside Sign", + "6042113507581755979": "Stellar Rocket", + "6005880141270483700": "Jolly Chimp", + "5998981470310368313": "Moon Pendant", + "5933937398953018107": "Ionic Dryer", + "5870972044522291836": "Input Key", + "5895518353849582541": "Mighty Arm", + "6005797617768858105": "Artisan Brick", + "5960747083030856414": "Clover Pin", + "5870947077877400011": "Sky Stilettos", + "5895603153683874485": "Fresh Socks", + "6006064678835323371": "Happy Brownie", + "5900177027566142759": "Ice Cream", + "5773725897517433693": "Spring Basket", + "6005564615793050414": "Instant Ramen", + "6003456431095808759": "Faith Amulet", + "5935877878062253519": "Mousse Cake", + "5902339509239940491": "Bling Binky", + "5963238670868677492": "Money Pot", + "5933737850477478635": "Pretty Posy", + "5839094187366024301": "Khabib's Papakha", + "5882260270843168924": "UFC Strike", + "5830340739074097859": "Victory Medal", + "5999116401002939514": "Rare Bird" +} \ No newline at end of file diff --git a/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py b/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py new file mode 100644 index 00000000..9ad3893c --- /dev/null +++ b/backend/core/src/core/migrations/versions/1772632462-abd20b1dfcfb-migrate_gift_collection_to_id.py @@ -0,0 +1,251 @@ +"""migrate_gift_collection_to_id + +Revision ID: abd20b1dfcfb +Revises: 6c6b45ffe090 +Create Date: 2026-03-04 13:54:22.662223 + +""" +from typing import Sequence, Union +import json +from pathlib import Path + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "abd20b1dfcfb" +down_revision: Union[str, None] = "6c6b45ffe090" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +GIFT_COLLECTION_IDS_MAPPING_PATH = ( + Path(__file__).parent / "../data/gift-collection-ids-to-names.json" +) + + +def upgrade_schema() -> None: + # 1. Add new columns + op.add_column( + "gift_collection", + sa.Column("id", sa.BigInteger(), autoincrement=False, nullable=True), + ) + op.add_column( + "gift_collection", + sa.Column("blockchain_address", sa.String(length=255), nullable=True), + ) + op.add_column( + "gift_collection", + sa.Column("options", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + op.add_column( + "gift_unique", sa.Column("collection_id", sa.BigInteger(), nullable=True) + ) + op.add_column( + "telegram_chat_gift_collection", + sa.Column("collection_id", sa.BigInteger(), nullable=True), + ) + + # 2. Drop foreign keys referencing `gift_collection.slug` + op.drop_constraint( + "gift_unique_collection_slug_fkey", "gift_unique", type_="foreignkey" + ) + op.drop_constraint( + "telegram_chat_gift_collection_collection_slug_fkey", + "telegram_chat_gift_collection", + type_="foreignkey", + ) + + +def migrate_data_upgrade() -> None: + # 3. Data Migration + with open(GIFT_COLLECTION_IDS_MAPPING_PATH, "r") as f: + mapping = json.load(f) + + connection = op.get_bind() + for col_id, name in mapping.items(): + connection.execute( + sa.text( + 'UPDATE gift_collection SET id = :id, options = \'{"models": [], "backdrops": [], "patterns": []}\'::jsonb WHERE title = :name' + ), + {"id": int(col_id), "name": name}, + ) + + # Update new fields in referencing tables + connection.execute( + sa.text( + "UPDATE gift_unique SET collection_id = gc.id FROM gift_collection gc WHERE gift_unique.collection_slug = gc.slug" + ) + ) + connection.execute( + sa.text( + "UPDATE telegram_chat_gift_collection SET collection_id = gc.id FROM gift_collection gc WHERE telegram_chat_gift_collection.collection_slug = gc.slug" + ) + ) + + +def cleanup_schema_upgrade() -> None: + connection = op.get_bind() + # Delete unmapped collections and their relations + connection.execute( + sa.text( + "DELETE FROM gift_unique WHERE collection_slug IN (SELECT slug FROM gift_collection WHERE id IS NULL)" + ) + ) + connection.execute( + sa.text( + "DELETE FROM telegram_chat_gift_collection WHERE collection_slug IN (SELECT slug FROM gift_collection WHERE id IS NULL)" + ) + ) + connection.execute(sa.text("DELETE FROM gift_collection WHERE id IS NULL")) + + # 4. Remove old PK & slug columns + op.drop_constraint("gift_collection_pkey", "gift_collection", type_="primary") + op.drop_column("gift_collection", "slug") + op.drop_column("gift_unique", "collection_slug") + op.drop_column("telegram_chat_gift_collection", "collection_slug") + + # 5. Alter columns non-nullable where needed + op.alter_column("gift_collection", "id", nullable=False) + op.alter_column("gift_collection", "options", nullable=False) + op.alter_column("gift_unique", "collection_id", nullable=False) + + # 6. Add new PK and foreign keys + op.create_primary_key("gift_collection_pkey", "gift_collection", ["id"]) + + op.create_foreign_key( + "gift_unique_collection_id_fkey", + "gift_unique", + "gift_collection", + ["collection_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + "telegram_chat_gift_collection_collection_id_fkey", + "telegram_chat_gift_collection", + "gift_collection", + ["collection_id"], + ["id"], + ondelete="CASCADE", + ) + + +BACKUP_TABLES = [ + ("gift_collection", "_backup_gift_collection"), + ("gift_unique", "_backup_gift_unique"), + ("telegram_chat_gift_collection", "_backup_telegram_chat_gift_collection"), +] + + +def backup_tables() -> None: + connection = op.get_bind() + for source, backup in BACKUP_TABLES: + connection.execute(sa.text(f"DROP TABLE IF EXISTS {backup}")) + connection.execute(sa.text(f"CREATE TABLE {backup} AS SELECT * FROM {source}")) + + +def drop_backup_tables() -> None: + connection = op.get_bind() + for _, backup in BACKUP_TABLES: + connection.execute(sa.text(f"DROP TABLE IF EXISTS {backup}")) + + +def upgrade() -> None: + backup_tables() + upgrade_schema() + migrate_data_upgrade() + cleanup_schema_upgrade() + + +def downgrade_schema() -> None: + # 1. Add back columns + op.add_column( + "telegram_chat_gift_collection", + sa.Column( + "collection_slug", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=True, + ), + ) + op.add_column( + "gift_unique", + sa.Column( + "collection_slug", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=True, + ), + ) + op.add_column( + "gift_collection", + sa.Column("slug", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + + # 2. Drop FKs and new PK + op.drop_constraint( + "telegram_chat_gift_collection_collection_id_fkey", + "telegram_chat_gift_collection", + type_="foreignkey", + ) + op.drop_constraint( + "gift_unique_collection_id_fkey", "gift_unique", type_="foreignkey" + ) + op.drop_constraint("gift_collection_pkey", "gift_collection", type_="primary") + + +def migrate_data_downgrade() -> None: + # 3. Data migration + connection = op.get_bind() + connection.execute( + sa.text("UPDATE gift_collection SET slug = REPLACE(LOWER(title), ' ', '-')") + ) + + connection.execute( + sa.text( + "UPDATE gift_unique SET collection_slug = gc.slug FROM gift_collection gc WHERE gift_unique.collection_id = gc.id" + ) + ) + connection.execute( + sa.text( + "UPDATE telegram_chat_gift_collection SET collection_slug = gc.slug FROM gift_collection gc WHERE telegram_chat_gift_collection.collection_id = gc.id" + ) + ) + + +def cleanup_schema_downgrade() -> None: + op.alter_column("gift_collection", "slug", nullable=False) + op.alter_column("gift_unique", "collection_slug", nullable=False) + + op.create_primary_key("gift_collection_pkey", "gift_collection", ["slug"]) + op.create_foreign_key( + "telegram_chat_gift_collection_collection_slug_fkey", + "telegram_chat_gift_collection", + "gift_collection", + ["collection_slug"], + ["slug"], + ondelete="CASCADE", + ) + op.create_foreign_key( + "gift_unique_collection_slug_fkey", + "gift_unique", + "gift_collection", + ["collection_slug"], + ["slug"], + ondelete="CASCADE", + ) + + op.drop_column("telegram_chat_gift_collection", "collection_id") + op.drop_column("gift_unique", "collection_id") + op.drop_column("gift_collection", "id") + op.drop_column("gift_collection", "blockchain_address") + op.drop_column("gift_collection", "options") + + +def downgrade() -> None: + downgrade_schema() + migrate_data_downgrade() + cleanup_schema_downgrade() + drop_backup_tables() diff --git a/backend/core/src/core/migrations/versions/1780515016-1c826351ffe6-merge_heads.py b/backend/core/src/core/migrations/versions/1780515016-1c826351ffe6-merge_heads.py new file mode 100644 index 00000000..44339310 --- /dev/null +++ b/backend/core/src/core/migrations/versions/1780515016-1c826351ffe6-merge_heads.py @@ -0,0 +1,24 @@ +"""merge heads + +Revision ID: 1c826351ffe6 +Revises: abd20b1dfcfb, 105b4511d5ca +Create Date: 2026-06-03 19:30:16.699890 + +""" + +from typing import Sequence, Union + + +# revision identifiers, used by Alembic. +revision: str = "1c826351ffe6" +down_revision: Union[str, None] = ("abd20b1dfcfb", "105b4511d5ca") +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/backend/core/src/core/models/gift.py b/backend/core/src/core/models/gift.py index e9fb6b8a..feab0225 100644 --- a/backend/core/src/core/models/gift.py +++ b/backend/core/src/core/models/gift.py @@ -1,6 +1,7 @@ import datetime from sqlalchemy import Integer, String, DateTime, ForeignKey, BigInteger +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import mapped_column from core.db import Base @@ -10,11 +11,17 @@ class GiftCollection(PricedEntityMixin): __tablename__ = "gift_collection" - slug = mapped_column(String(255), primary_key=True) + id = mapped_column(BigInteger, primary_key=True, autoincrement=False) title = mapped_column(String(255), nullable=False, unique=True) preview_url = mapped_column(String(255), nullable=True) supply = mapped_column(Integer, nullable=False, default=0) upgraded_count = mapped_column(Integer, nullable=False, default=0) + blockchain_address = mapped_column(String(255), nullable=True) + options = mapped_column( + JSONB, + nullable=False, + default=lambda: {"models": [], "backdrops": [], "patterns": []}, + ) last_updated = mapped_column( DateTime, nullable=False, @@ -27,8 +34,13 @@ class GiftUnique(Base): __tablename__ = "gift_unique" slug = mapped_column(String(255), primary_key=True) - collection_slug = mapped_column( - ForeignKey("gift_collection.slug", ondelete="CASCADE"), nullable=False + collection_id = mapped_column( + ForeignKey( + "gift_collection.id", + ondelete="CASCADE", + name="gift_unique_collection_id_fkey", + ), + nullable=False, ) telegram_owner_id = mapped_column( BigInteger, diff --git a/backend/core/src/core/models/rule.py b/backend/core/src/core/models/rule.py index f29f2786..832856a1 100644 --- a/backend/core/src/core/models/rule.py +++ b/backend/core/src/core/models/rule.py @@ -284,10 +284,14 @@ class TelegramChatStickerCollection(TelegramChatThresholdRuleBase): class TelegramChatGiftCollection(TelegramChatThresholdRuleBase): __tablename__ = "telegram_chat_gift_collection" - collection_slug = mapped_column( - ForeignKey("gift_collection.slug", ondelete="CASCADE"), + collection_id = mapped_column( + ForeignKey( + "gift_collection.id", + ondelete="CASCADE", + name="telegram_chat_gift_collection_collection_id_fkey", + ), nullable=True, - doc="Collection slug that will be used to check eligibility for the collection.", + doc="Collection ID that will be used to check eligibility for the collection.", ) collection: Mapped["GiftCollection"] = relationship( "GiftCollection", diff --git a/backend/core/src/core/services/gift/collection.py b/backend/core/src/core/services/gift/collection.py index 887e5ceb..c0fb5420 100644 --- a/backend/core/src/core/services/gift/collection.py +++ b/backend/core/src/core/services/gift/collection.py @@ -3,42 +3,47 @@ class GiftCollectionService(BaseService): - def get(self, slug: str) -> GiftCollection: + def get(self, id: int) -> GiftCollection: return ( - self.db_session.query(GiftCollection) - .filter(GiftCollection.slug == slug) - .one() + self.db_session.query(GiftCollection).filter(GiftCollection.id == id).one() ) - def get_all(self, slugs: list[str] | None = None) -> list[GiftCollection]: + def get_all(self, ids: list[int] | None = None) -> list[GiftCollection]: query = self.db_session.query(GiftCollection) - if slugs: - query = query.filter(GiftCollection.slug.in_(slugs)) - result = query.order_by(GiftCollection.slug).all() + if ids: + query = query.filter(GiftCollection.id.in_(ids)) + result = query.order_by(GiftCollection.title).all() return result - def find(self, slug: str) -> GiftCollection | None: + def get_all_ids(self) -> set[int]: + return {row[0] for row in self.db_session.query(GiftCollection.id).all()} + + def find(self, id: int) -> GiftCollection | None: return ( self.db_session.query(GiftCollection) - .filter(GiftCollection.slug == slug) + .filter(GiftCollection.id == id) .first() ) def create( self, - slug: str, + id: int, title: str, preview_url: str | None, supply: int | None, upgraded_count: int | None, + options: dict[str, list[str]] | None = None, ) -> GiftCollection: new_collection = GiftCollection( - slug=slug, + id=id, title=title, preview_url=preview_url, supply=supply, upgraded_count=upgraded_count, ) + if options is not None: + new_collection.options = options.model_dump() + self.db_session.add(new_collection) self.db_session.flush() @@ -46,17 +51,20 @@ def create( def update( self, - slug: str, + id: int, title: str, preview_url: str | None, supply: int | None, upgraded_count: int | None, + options: dict[str, list[str]] | None = None, ) -> GiftCollection: - collection = self.get(slug) + collection = self.get(id) collection.title = title collection.preview_url = preview_url collection.supply = supply collection.upgraded_count = upgraded_count + if options is not None: + collection.options = options.model_dump() self.db_session.flush() return collection diff --git a/backend/core/src/core/services/gift/item.py b/backend/core/src/core/services/gift/item.py index bb8d221a..5bee7e5e 100644 --- a/backend/core/src/core/services/gift/item.py +++ b/backend/core/src/core/services/gift/item.py @@ -1,6 +1,5 @@ from collections import namedtuple -from sqlalchemy import select, func from core.models.gift import GiftUnique from core.services.base import BaseService @@ -15,54 +14,35 @@ def get(self, slug: str) -> GiftUnique: def get_all( self, - collection_slug: str | None = None, + collection_id: int | None = None, telegram_user_id: int | None = None, - number_ge: int | None = None, - number_le: int | None = None, ) -> list[GiftUnique]: query = self.db_session.query(GiftUnique) - if collection_slug: - query = query.filter(GiftUnique.collection_slug == collection_slug) + if collection_id: + query = query.filter(GiftUnique.collection_id == collection_id) if telegram_user_id: query = query.filter(GiftUnique.telegram_owner_id == telegram_user_id) - if number_ge: - query = query.filter(GiftUnique.number >= number_ge) - if number_le: - query = query.filter(GiftUnique.number <= number_le) return query.order_by(GiftUnique.number).all() - def get_unique_options(self, collection_slug: str): - query = select( - func.array_agg(func.distinct(GiftUnique.model)).label("models"), - func.array_agg(func.distinct(GiftUnique.backdrop)).label("backdrops"), - func.array_agg(func.distinct(GiftUnique.pattern)).label("patterns"), - ).where(GiftUnique.collection_slug == collection_slug) - - result = self.db_session.execute(query).first() - - return { - "models": [m for m in (result.models or []) if m is not None], - "backdrops": [b for b in (result.backdrops or []) if b is not None], - "patterns": [p for p in (result.patterns or []) if p is not None], - } - def find(self, slug: str) -> GiftUnique | None: return self.db_session.query(GiftUnique).filter(GiftUnique.slug == slug).first() def create( self, slug: str, + collection_id: int, number: int, - model: str, - backdrop: str, - pattern: str, - telegram_owner_id: int | None, - blockchain_address: str | None, - owner_address: str | None, + model: str | None = None, + backdrop: str | None = None, + pattern: str | None = None, + telegram_owner_id: int | None = None, + blockchain_address: str | None = None, + owner_address: str | None = None, ) -> GiftUnique: new_unique = GiftUnique( slug=slug, + collection_id=collection_id, number=number, model=model, backdrop=backdrop, @@ -79,12 +59,12 @@ def update( self, slug: str, number: int, - model: str, - backdrop: str, - pattern: str, - telegram_owner_id: int | None, - blockchain_address: str | None, - owner_address: str | None, + model: str | None = None, + backdrop: str | None = None, + pattern: str | None = None, + telegram_owner_id: int | None = None, + blockchain_address: str | None = None, + owner_address: str | None = None, ) -> GiftUnique: unique = self.get(slug) unique.number = number @@ -101,7 +81,7 @@ def update( def update_ownership( self, slug: str, - telegram_owner_id: int, + telegram_owner_id: int | None, blockchain_address: str | None, owner_address: str | None, ) -> GiftUnique: diff --git a/backend/core/src/core/services/ton.py b/backend/core/src/core/services/ton.py index 4393710c..517f5709 100644 --- a/backend/core/src/core/services/ton.py +++ b/backend/core/src/core/services/ton.py @@ -91,12 +91,12 @@ def get_ton_price(self) -> float | None: # Handle Redis service unavailability gracefully raw_value = self.redis_service.get(TON_PRICE_CACHE_KEY) if raw_value is None: - logger.warning("TON price is not cached yet.") + logger.warning("GRAM price is not cached yet.") return None return float(raw_value) except Exception as e: - logger.error("Failed to get TON price: %s", e) + logger.error("Failed to get GRAM price: %s", e) return None def set_ton_price(self, price: float) -> None: diff --git a/backend/core/src/core/services/user.py b/backend/core/src/core/services/user.py index fc1dc255..fc44ad3a 100644 --- a/backend/core/src/core/services/user.py +++ b/backend/core/src/core/services/user.py @@ -1,4 +1,5 @@ -from typing import Iterable, Any +import logging +from typing import Any, Iterable from sqlalchemy.orm import joinedload, load_only, QueryableAttribute @@ -6,6 +7,8 @@ from core.models.user import User from core.services.base import BaseService +logger = logging.getLogger(__name__) + class UserService(BaseService): def get_all( @@ -22,6 +25,25 @@ def get_all( return query.all() + def get_all_telegram_ids(self) -> list[int]: + return [ + u.telegram_id + for u in self.db_session.query(User.telegram_id) + .filter(User.telegram_id.is_not(None)) + .filter(User.is_blocked.is_(False)) + .all() + ] + + def mark_as_blocked(self, telegram_ids: list[int]) -> None: + if not telegram_ids: + return + + logger.info(f"Marking {len(telegram_ids)} users as blocked.") + self.db_session.query(User).filter(User.telegram_id.in_(telegram_ids)).update( + {"is_blocked": True}, synchronize_session=False + ) + self.db_session.flush() + def get_by_telegram_id(self, telegram_id: int) -> User: return ( self.db_session.query(User) diff --git a/backend/core/src/core/settings.py b/backend/core/src/core/settings.py index 9e0dbba0..69b43d0b 100644 --- a/backend/core/src/core/settings.py +++ b/backend/core/src/core/settings.py @@ -52,16 +52,6 @@ def blacklisted_wallets(self): return self._blacklisted_wallets - _whitelisted_gift_collections: list[str] | None = None - - @property - def whitelisted_gift_collections(self): - if self._whitelisted_gift_collections is None: - with open(STATIC_PATH / "whitelisted_gift_collections.txt") as f: - self._whitelisted_gift_collections = f.read().splitlines() - - return self._whitelisted_gift_collections - beat_schedule_filename: str = "/tmp/celerybeat-schedule" env: str = "development" diff --git a/backend/core/src/core/static/whitelisted_gift_collections.txt b/backend/core/src/core/static/whitelisted_gift_collections.txt deleted file mode 100644 index 9235fed1..00000000 --- a/backend/core/src/core/static/whitelisted_gift_collections.txt +++ /dev/null @@ -1,23 +0,0 @@ -PlushPepe -DurovsCap -PreciousPeach -SwissWatch -BondedRing -GemSignet -ScaredCat -LootBag -NekoHelmet -IonGem -TopHat -MiniOscar -MagicPotion -AstralShard -VintageCigar -ElectricSkull -SignetRing -LowRider -WestsideSign -CupidCharm -HeartLocket -SnoopDogg -KhabibsPapakha diff --git a/backend/core/src/core/utils/gift.py b/backend/core/src/core/utils/gift.py index f5b99d3b..49660940 100644 --- a/backend/core/src/core/utils/gift.py +++ b/backend/core/src/core/utils/gift.py @@ -15,7 +15,7 @@ def find_relevant_gift_items( This function iterates over a list of gift items and filters them according to the attributes defined in the provided rule object. Attributes such as - collection_slug, model, backdrop, and pattern are used to determine the + collection_id, model, backdrop, and pattern are used to determine the relevance of each item. Items that match the defined criteria are appended to a resulting list, which is then returned. @@ -32,10 +32,7 @@ def find_relevant_gift_items( logger.warning(f"Trying to get gifts by category {rule.category!r}.") continue - if ( - rule.collection_slug is not None - and rule.collection_slug != item.collection_slug - ): + if rule.collection_id is not None and rule.collection_id != item.collection_id: continue if rule.model is not None and rule.model != item.model: diff --git a/backend/indexer_gifts/actions/api.py b/backend/indexer_gifts/actions/api.py new file mode 100644 index 00000000..3ce4a35b --- /dev/null +++ b/backend/indexer_gifts/actions/api.py @@ -0,0 +1,75 @@ +import logging + +from sqlalchemy.orm import Session + +from core.constants import GIFT_COLLECTIONS_METADATA_KEY +from core.actions.base import BaseAction +from core.services.gift.collection import GiftCollectionService +from core.services.superredis import RedisService +from indexer_gifts.indexers.gift_changes import GiftChangesIndexer + + +logger = logging.getLogger(__name__) + + +class IndexerGiftChangesAction(BaseAction): + def __init__(self, db_session: Session) -> None: + super().__init__(db_session) + self.service = GiftCollectionService(db_session) + self.indexer = GiftChangesIndexer() + self.redis_service = RedisService() + + async def index_all(self) -> None: + """ + Fetches all gift collections from the changes.tg API via `GiftChangesIndexer` + and synchronizes them to the database using `GiftCollectionService`. + Updates existing entries, and creates missing ones. + """ + logger.info("Starting synchronization of gift collections from changes.tg API") + + try: + collections_dtos = await self.indexer.fetch_all_collections() + finally: + await self.indexer.close() + + logger.info( + f"Fetched {len(collections_dtos)} valid collections from changes.tg API" + ) + + for collection_dto in collections_dtos: + parsed_id = int(collection_dto.id) + + existing_record = self.service.find(parsed_id) + + if not existing_record: + gift_collection = self.service.create( + id=parsed_id, + title=collection_dto.title, + preview_url=collection_dto.preview_url, + supply=collection_dto.supply, # defaults to 0 from the indexer + upgraded_count=collection_dto.upgraded_count, # defaults to 0 + options=collection_dto.options, + ) + logger.info( + f"Created gift collection {gift_collection.title!r} ({gift_collection.id}) successfully." + ) + else: + logger.debug( + f"Gift collection {collection_dto.title!r} ({parsed_id}) already exists. Updating properties..." + ) + # Re-use the existing supply and upgraded_count to prevent overwriting with 0 + gift_collection = self.service.update( + id=parsed_id, + title=collection_dto.title, + preview_url=collection_dto.preview_url, + supply=existing_record.supply, + upgraded_count=existing_record.upgraded_count, + options=collection_dto.options, + ) + logger.info( + f"Updated gift collection {gift_collection.title!r} ({gift_collection.id}) successfully." + ) + + # Reset the metadata cache as new items/options appear + self.redis_service.delete(GIFT_COLLECTIONS_METADATA_KEY) + logger.info("Synchronization of gift collections finished successfully.") diff --git a/backend/indexer_gifts/actions/collection.py b/backend/indexer_gifts/actions/collection.py deleted file mode 100644 index 5a5633c6..00000000 --- a/backend/indexer_gifts/actions/collection.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging -from pathlib import Path - -from sqlalchemy.orm import Session - -from core.actions.base import BaseAction -from core.constants import GIFT_COLLECTIONS_METADATA_KEY -from core.dtos.gift.collection import GiftCollectionDTO -from core.services.gift.collection import GiftCollectionService -from core.services.superredis import RedisService -from indexer_gifts.indexers.collection import GiftCollectionIndexer - - -logger = logging.getLogger(__name__) - - -class IndexerGiftCollectionAction(BaseAction): - def __init__(self, db_session: Session, session_path: Path) -> None: - super().__init__(db_session) - self.service = GiftCollectionService(db_session) - self.indexer = GiftCollectionIndexer(session_path=session_path) - self.redis_service = RedisService() - - async def index(self, slug: str) -> GiftCollectionDTO: - gift_collection_dto = await self.indexer.index(slug) - - if not self.service.find(slug): - gift_collection = self.service.create( - slug=gift_collection_dto.slug, - title=gift_collection_dto.title, - preview_url=gift_collection_dto.preview_url, - supply=gift_collection_dto.supply, - upgraded_count=gift_collection_dto.upgraded_count, - ) - logger.info( - f"Created gift collection {gift_collection.slug!r} successfully." - ) - else: - logger.info( - f"Gift collection {gift_collection_dto.slug!r} already exists. Updating..." - ) - gift_collection = self.service.update( - slug=gift_collection_dto.slug, - title=gift_collection_dto.title, - preview_url=gift_collection_dto.preview_url, - supply=gift_collection_dto.supply, - upgraded_count=gift_collection_dto.upgraded_count, - ) - logger.info( - f"Updated gift collection {gift_collection.slug!r} successfully." - ) - # Reset the metadata cache as new items appear - self.redis_service.delete(GIFT_COLLECTIONS_METADATA_KEY) - return GiftCollectionDTO.from_orm(gift_collection) diff --git a/backend/indexer_gifts/actions/item.py b/backend/indexer_gifts/actions/item.py deleted file mode 100644 index 60fa7950..00000000 --- a/backend/indexer_gifts/actions/item.py +++ /dev/null @@ -1,158 +0,0 @@ -import datetime -import logging -from pathlib import Path -from typing import AsyncGenerator - -from sqlalchemy.orm import Session - -from core.actions.base import BaseAction -from core.constants import GIFT_COLLECTIONS_METADATA_KEY -from core.models.gift import GiftUnique, GiftCollection -from core.services.gift.collection import GiftCollectionService -from core.services.gift.item import GiftUniqueService -from core.services.superredis import RedisService -from indexer_gifts.indexers.item import GiftUniqueIndexer -from indexer_gifts.settings import gifts_indexer_settings - -logger = logging.getLogger(__name__) - - -class IndexerGiftUniqueAction(BaseAction): - def __init__(self, db_session: Session, session_path: Path) -> None: - super().__init__(db_session) - self.collection_service = GiftCollectionService(db_session) - self.service = GiftUniqueService(db_session) - self.redis_service = RedisService() - self.indexer = GiftUniqueIndexer(session_path=session_path) - - async def index_all(self) -> AsyncGenerator[set[int], None]: - """ - Indexes all unique items in the database. - """ - logger.info("Starting indexing all unique items...") - for collection in self.collection_service.get_all( - slugs=gifts_indexer_settings.whitelisted_gift_collections - ): - yield await self._index(collection, start=1, stop=collection.upgraded_count) - logger.info("Finished indexing all unique items.") - - async def _index( - self, collection: GiftCollection, start: int | None, stop: int | None - ) -> set[int]: - """ - Indexes and processes a collection of unique items while managing creation, - updates, and targeted owners. - This function interacts with a database to synchronize the collection data - and updates Redis caching as necessary. - - - :param collection: The gift collection object to be indexed. - :param start: The starting index for the indexing process. Defaults to 1. - :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. - :return: A set of Telegram owner IDs that require targeted actions due to - ownership changes or updates. - """ - if start is None: - start = 1 - - if stop is None: - stop = collection.upgraded_count - - existing_items = { - item.slug: item - for item in self.service.get_all( - collection_slug=collection.slug, - number_ge=start, - number_le=stop, - ) - } - logger.info( - f"Found existing {len(existing_items)} unique items " - f"for collection {collection.slug!r} and starting from index {start} to {stop}..." - ) - targeted_telegram_owner_ids = set() - - # Iterate over batches and process items - async for batch in self.indexer.index_collection_items( - collection_slug=collection.slug, - start=start, - stop=stop, - ): - to_create = [] - to_update = [] - for item in batch: - if existing_item := existing_items.get(item.slug): - # Update record only if necessary - if any( - ( - item.telegram_owner_id != existing_item.telegram_owner_id, - item.owner_address != existing_item.owner_address, - ) - ): - # Ignore if the owner was previously hidden - if isinstance(existing_item.telegram_owner_id, int): - # Store Telegram ID of the previous owner to perform and actions on losing ownership if needed - targeted_telegram_owner_ids.add( - existing_item.telegram_owner_id - ) - to_update.append( - { - "slug": item.slug, - "telegram_owner_id": item.telegram_owner_id, - "owner_address": item.owner_address, - "blockchain_address": item.blockchain_address, - "last_updated": datetime.datetime.now(tz=datetime.UTC), - } - ) - else: - logger.debug( - f"No changes detected for item {item.slug!r} in collection {collection.slug!r}. Skipping." - ) - else: - to_create.append( - GiftUnique( - slug=item.slug, - collection_slug=collection.slug, - model=item.model, - backdrop=item.backdrop, - number=item.number, - pattern=item.pattern, - telegram_owner_id=item.telegram_owner_id, - owner_address=item.owner_address, - blockchain_address=item.blockchain_address, - last_updated=datetime.datetime.now(tz=datetime.UTC), - ) - ) - if to_create: - # Cache has to be cleared as new metadata could appear. - # It could be extended to clear only when new metadata options appear. - self.redis_service.delete(GIFT_COLLECTIONS_METADATA_KEY) - - self.db_session.bulk_save_objects(to_create) - logger.info( - f"Created {len(to_create)} new unique items for collection {collection.slug!r}." - ) - self.db_session.bulk_update_mappings(GiftUnique, to_update) - logger.info( - f"Updated {len(to_update)} existing unique items for collection {collection.slug!r}." - ) - self.db_session.commit() - - return targeted_telegram_owner_ids - - async def index( - self, slug: str, start: int | None = None, stop: int | None = None - ) -> set[int]: - """ - Processes a collection of items by indexing, updating, or creating unique records. It also - tracks and returns Telegram IDs of previous owners of updated items for further actions. - - :param slug: A unique identifier for the collection being processed. - :param start: The starting index for the indexing process. Defaults to 1. - :param stop: The ending index for the indexing process. Defaults to the number of upgraded items. - - :return: A set of Telegram IDs corresponding to previous owners of the updated - items who need further actions. - """ - collection = self.collection_service.get(slug) - return await self._index(collection, start=start, stop=stop) diff --git a/backend/indexer_gifts/actions/user.py b/backend/indexer_gifts/actions/user.py new file mode 100644 index 00000000..43110f8d --- /dev/null +++ b/backend/indexer_gifts/actions/user.py @@ -0,0 +1,169 @@ +import logging +from sqlalchemy.orm import Session + +from core.actions.base import BaseAction +from core.constants import UPDATED_GIFT_USER_IDS +from core.services.gift.collection import GiftCollectionService +from core.services.gift.item import GiftUniqueService +from core.services.superredis import RedisService +from core.services.user import UserService +from indexer_gifts.indexers.bot_api import BotApiGiftIndexer + +logger = logging.getLogger(__name__) + + +class IndexerUserGiftAction(BaseAction): + def __init__(self, db_session: Session): + super().__init__(db_session) + self.gift_service = GiftUniqueService(db_session) + self.gift_collection_service = GiftCollectionService(db_session) + self.redis_service = RedisService() + self.user_service = UserService(db_session) + + async def sync_single_user_gifts(self, telegram_user_id: int) -> None: + """Public method for single-user sync, manages the indexer lifecycle.""" + async with BotApiGiftIndexer() as indexer: + try: + await self._sync_user_gifts(telegram_user_id, indexer=indexer) + except Exception as e: + if "user not found" in str(e).lower(): + self.user_service.mark_as_blocked([telegram_user_id]) + raise + + async def sync_users_batch( + self, + telegram_user_ids: list[int], + existing_collection_ids: set[int] | None = None, + ) -> None: + """Public method for batch sync, manages a single indexer across multiple users.""" + blocked_user_ids = [] + async with BotApiGiftIndexer() as indexer: + for user_id in telegram_user_ids: + try: + await self._sync_user_gifts( + user_id, + indexer=indexer, + existing_collection_ids=existing_collection_ids, + ) + except Exception as e: + logger.error(f"Error refreshing gifts for user {user_id}: {e}") + if "user not found" in str(e).lower(): + blocked_user_ids.append(user_id) + + if blocked_user_ids: + self.user_service.mark_as_blocked(blocked_user_ids) + + async def _sync_user_gifts( + self, + telegram_user_id: int, + indexer: BotApiGiftIndexer, + existing_collection_ids: set[int] | None = None, + ) -> None: + """ + Internal business logic to sync unique gifts for a user using the provided indexer. + """ + logger.info(f"Starting gifts synchronizing for user {telegram_user_id}") + current_owned_slugs = set() + lost_ownership_user_ids = set() + created_count = 0 + updated_count = 0 + lost_count = 0 + + # Prefetch collection IDs to avoid N+1 queries if not provided by batch caller + if existing_collection_ids is None: + existing_collection_ids = self.gift_collection_service.get_all_ids() + + # 1. Fetch current gifts from Telegram API + async for owned_gift in indexer.iter_user_gifts(telegram_user_id): + ug = owned_gift.gift + current_owned_slugs.add(ug.name) + + # Check if it exists + existing = self.gift_service.find(ug.name) + + if existing: + # Only update if ownership actually changed + if existing.telegram_owner_id != telegram_user_id: + if existing.telegram_owner_id: + lost_ownership_user_ids.add(existing.telegram_owner_id) + + self.gift_service.update_ownership( + slug=ug.name, + telegram_owner_id=telegram_user_id, + blockchain_address=existing.blockchain_address, + owner_address=existing.owner_address, + ) + updated_count += 1 + else: + # Ensure collection exists using in-memory cache + collection_id = int(ug.gift_id) + if collection_id not in existing_collection_ids: + logger.info( + f"Collection {collection_id} missing, creating stub collection." + ) + self.gift_collection_service.create( + id=collection_id, + title=ug.base_name or f"Gift {collection_id}", + preview_url=None, + supply=0, + upgraded_count=0, + ) + existing_collection_ids.add(collection_id) + + # Extract string identifiers for visual attributes if available + model_name = ( + getattr(ug.model, "name", None) + if getattr(ug, "model", None) + else None + ) + backdrop_name = ( + getattr(ug.backdrop, "name", None) + if getattr(ug, "backdrop", None) + else None + ) + pattern_name = ( + getattr(ug.symbol, "name", None) + if getattr(ug, "symbol", None) + else None + ) + + # Create new gift + self.gift_service.create( + slug=ug.name, + collection_id=collection_id, + number=ug.number, + model=model_name, + backdrop=backdrop_name, + pattern=pattern_name, + telegram_owner_id=telegram_user_id, + blockchain_address=None, + owner_address=None, + ) + created_count += 1 + + # 2. Fetch all gifts currently assigned to the user in the database + db_gifts = self.gift_service.get_all(telegram_user_id=telegram_user_id) + for db_gift in db_gifts: + if db_gift.slug not in current_owned_slugs: + # User no longer owns this gift + self.gift_service.update_ownership( + slug=db_gift.slug, + telegram_owner_id=None, + blockchain_address=db_gift.blockchain_address, + owner_address=db_gift.owner_address, + ) + lost_ownership_user_ids.add(telegram_user_id) + lost_count += 1 + + # 3. Add users who lost ownership to the Redis set for community manager checking + if lost_ownership_user_ids: + logger.info( + f"Adding {len(lost_ownership_user_ids)} users to Redis for community manager check due to lost gifts." + ) + for uid in lost_ownership_user_ids: + self.redis_service.add_to_set(UPDATED_GIFT_USER_IDS, str(uid)) + + logger.info( + f"Finished gifts synchronizing for user {telegram_user_id}. " + f"Created: {created_count}, Updated: {updated_count}, Lost: {lost_count}." + ) diff --git a/backend/indexer_gifts/celery_app.py b/backend/indexer_gifts/celery_app.py index 939c8583..aedaabba 100644 --- a/backend/indexer_gifts/celery_app.py +++ b/backend/indexer_gifts/celery_app.py @@ -16,6 +16,8 @@ def create_app() -> Celery: "result_expires": 300, # 5 minutes "include": ["indexer_gifts.tasks"], "worker_concurrency": gifts_indexer_settings.worker_concurrency, + "broker_transport_options": {"queue_order_strategy": "priority"}, + "worker_prefetch_multiplier": 1, } ) return _app diff --git a/backend/indexer_gifts/indexers/bot_api.py b/backend/indexer_gifts/indexers/bot_api.py new file mode 100644 index 00000000..3732e0ea --- /dev/null +++ b/backend/indexer_gifts/indexers/bot_api.py @@ -0,0 +1,74 @@ +import logging +import asyncio +from typing import AsyncGenerator + +from aiogram import Bot +from aiogram.client.default import DefaultBotProperties +from aiogram.types import OwnedGifts, OwnedGiftUnique + +from core.settings import core_settings + +logger = logging.getLogger(__name__) + + +class BotApiGiftIndexer: + def __init__(self): + # Initialize Bot with the token from core settings + self.bot = Bot( + token=core_settings.telegram_bot_token, + default=DefaultBotProperties(parse_mode="HTML"), + ) + + async def iter_user_gifts( + self, telegram_user_id: int + ) -> AsyncGenerator[OwnedGiftUnique, None]: + """ + Yields OwnedGiftUnique objects owned by the user via pagination. + Handles rate limits or pagination as needed. + """ + offset = "" + page_num = 1 + logger.info(f"Starting to fetch gifts for user {telegram_user_id}...") + while True: + try: + logger.debug( + f"Fetching page {page_num} for user {telegram_user_id} (offset: {offset!r})" + ) + # get_user_gifts is available in aiogram 3.24.0 + user_gifts: OwnedGifts = await self.bot.get_user_gifts( + user_id=telegram_user_id, offset=offset, limit=100 + ) + + logger.info( + f"Fetched {len(user_gifts.gifts)} gifts for user {telegram_user_id} on page {page_num}." + ) + + # Yield only unique gifts from the current page + for gift in user_gifts.gifts: + if isinstance(gift, OwnedGiftUnique): + yield gift + + # Check if there are more pages + if not user_gifts.next_offset: + logger.info( + f"Finished fetching all gifts for user {telegram_user_id}. Total count: {user_gifts.total_count}" + ) + break + + offset = user_gifts.next_offset + page_num += 1 + + except Exception as e: + logger.error(f"Error fetching gifts for user {telegram_user_id}: {e}") + raise + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def close(self): + await self.bot.session.close() + # Allow underlying aiohttp SSL connections to close gracefully + await asyncio.sleep(0.250) diff --git a/backend/indexer_gifts/indexers/collection.py b/backend/indexer_gifts/indexers/collection.py deleted file mode 100644 index 405bcf2f..00000000 --- a/backend/indexer_gifts/indexers/collection.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -from pathlib import Path -from tempfile import NamedTemporaryFile - -from telethon.errors import BadRequestError - -from core.dtos.gift.collection import GiftCollectionDTO -from core.exceptions.gift import GiftCollectionNotExistsError -from core.services.cdn import CDNService -from core.services.supertelethon import TelethonService - -logger = logging.getLogger(__name__) - - -class GiftCollectionIndexer: - def __init__(self, session_path: Path) -> None: - self.telethon_service = TelethonService(session_path=session_path) - self.cdn_service = CDNService() - - async def index(self, slug: str) -> GiftCollectionDTO: - logger.info(f"Indexing gift collection {slug!r}...") - await self.telethon_service.start() - try: - first_unique_gift = await self.telethon_service.index_gift( - slug=slug, - number=1, - ) - except BadRequestError as exc: - logger.exception(f"Gift Collection with slug {slug!r} not found.") - raise GiftCollectionNotExistsError( - f"Gift Collection with slug {slug!r} not found." - ) from exc - logger.info(f"Indexed gift collection {slug!r}.") - logger.info(f"Downloading unique gift thumbnail for {slug!r}...") - with NamedTemporaryFile(mode="w+b", delete=True) as tmp_file: - file_name = await self.telethon_service.download_unique_gift_thumbnail( - entity=first_unique_gift, - target_location=tmp_file, - ) - await self.cdn_service.upload_file( - file_path=tmp_file.name, object_name=file_name - ) - logger.info( - f"Downloaded unique gift thumbnail for {slug!r}: {file_name!r}.." - ) - # Free session for the next process - await self.telethon_service.stop() - return GiftCollectionDTO.from_telethon( - slug=slug, - obj=first_unique_gift, - preview_url=file_name, - ) diff --git a/backend/indexer_gifts/indexers/gift_changes.py b/backend/indexer_gifts/indexers/gift_changes.py new file mode 100644 index 00000000..3690d662 --- /dev/null +++ b/backend/indexer_gifts/indexers/gift_changes.py @@ -0,0 +1,79 @@ +import httpx +from pydantic import ValidationError + +from core.dtos.gift.collection import ( + GiftCollectionMetadataDTO, + GiftCollectionOptionsDTO, +) + + +class GiftChangesIndexer: + def __init__(self) -> None: + self.client = httpx.AsyncClient() + self.base_url = "https://api.changes.tg" + self.cdn_url = "https://cdn.changes.tg/gifts" + + async def fetch_all_collections(self) -> list[GiftCollectionMetadataDTO]: + """ + Fetches all available collection IDs, then fetches specifics for each ID. + Constructs and validates the GiftCollectionMetadataDTO for each collection. + Returns a list of validated collections. + """ + ids_url = f"{self.base_url}/ids" + response = await self.client.get(ids_url) + response.raise_for_status() + + # The API returns {"5983471780763796287":"Santa Hat", ...} + ids_data: dict[str, str] = response.json() + collections: list[GiftCollectionMetadataDTO] = [] + + for gift_id, title in ids_data.items(): + collection = await self.fetch_single_collection(gift_id, title) + if collection: + collections.append(collection) + + return collections + + async def fetch_single_collection( + self, gift_id: str, title: str + ) -> GiftCollectionMetadataDTO | None: + """ + Fetches details for a single gift ID and constructs its Metadata DTO. + """ + gift_url = f"{self.base_url}/gift/{gift_id}" + response = await self.client.get(gift_url) + + if response.status_code != 200: + return None + + data = response.json() + + # Parse attributes + models = [item["name"] for item in data.get("models", [])] + backdrops = [item["name"] for item in data.get("backdrops", [])] + symbols = [item["name"] for item in data.get("symbols", [])] + + preview_url = f"{self.cdn_url}/originals/{gift_id}/Original.png" + + try: + # We construct supply and upgraded as 0 right now + # as it will be safely overwritten later if necessary + dto = GiftCollectionMetadataDTO( + id=gift_id, + title=title, + preview_url=preview_url, + supply=0, + upgraded_count=0, + options=GiftCollectionOptionsDTO( + models=models, + backdrops=backdrops, + patterns=symbols, + ), + ) + return dto + except ValidationError: + # If the properties fail validation, we return None and skip this element + return None + + async def close(self) -> None: + await self.client.aclose() diff --git a/backend/indexer_gifts/indexers/item.py b/backend/indexer_gifts/indexers/item.py deleted file mode 100644 index 9ad39e92..00000000 --- a/backend/indexer_gifts/indexers/item.py +++ /dev/null @@ -1,110 +0,0 @@ -import logging -from pathlib import Path -from typing import AsyncGenerator - -from telethon.errors import RPCError - -from core.dtos.gift.item import GiftUniqueDTO -from core.services.supertelethon import TelethonService -from indexer_gifts.settings import gifts_indexer_settings -from indexer_gifts.utils import parse_collection_slug_from_gift_slug - -logger = logging.getLogger(__name__) - - -class GiftUniqueIndexer: - def __init__(self, session_path: Path) -> None: - self.telethon_service = TelethonService(session_path=session_path) - - async def index_collection_items( - self, collection_slug: str, start: int, stop: int - ) -> AsyncGenerator[list[GiftUniqueDTO], None]: - """ - Indexes all gifts from a specific collection using the telethon service. This process - involves fetching a specific number of gifts from a collection, converting them into - GiftUniqueDTO objects, and yielding them in batches of a defined size. - - :param collection_slug: The unique identifier of the collection to be indexed. - :param start: The first index of the gifts to be indexed from the collection. - :param stop: The total number of gifts to be indexed from the collection. - :return: An asynchronous generator yielding lists of GiftUniqueDTO objects. - """ - await self.telethon_service.start() - try: - entities = [] - for num in range( - start, stop + 1, gifts_indexer_settings.telegram_batch_request_size - ): - gifts = await self.telethon_service.index_gifts_batch( - slugs=[ - f"{collection_slug}-{gift_id}" - for i in range( - gifts_indexer_settings.telegram_batch_request_size - ) - if (gift_id := num + i) <= stop - ] - ) - - logger.debug( - f"Indexed gifts {num}...{num + gifts_indexer_settings.telegram_batch_request_size - 1} " - f"for {collection_slug!r}." - ) - - entities.extend( - [ - GiftUniqueDTO.from_telethon( - collection_slug=collection_slug, - obj=gift, - ) - for gift in gifts - ] - ) - - if ( - len(entities) - >= gifts_indexer_settings.telegram_batch_processing_size - ): - logger.info(f"Indexed {num} unique gifts for {collection_slug!r}.") - yield entities - entities = [] - - yield entities - except RPCError: - logger.exception( - f"Failed to index gifts for collection {collection_slug!r}" - ) - finally: - # Free session for the next process - await self.telethon_service.stop() - - async def index_user_gifts( - self, - telegram_user_id: int, - ) -> list[GiftUniqueDTO]: - """ - Indexes and retrieves a list of unique gifts associated with a specific Telegram user. - - The function initiates a Telethon service session, fetches the gifts for a given - Telegram user, and processes them to extract and organize unique gift details. - If a gift's collection slug cannot be parsed from its slug, it logs a warning - message and skips processing for that specific gift. - - :param telegram_user_id: The unique ID of the Telegram user whose gifts are - being indexed. - :return: A list of GiftUniqueDTO objects representing unique gifts associated - with the given Telegram user. - """ - await self.telethon_service.start() - gifts = await self.telethon_service.index_user_gifts(telegram_user_id) - entities = [] - for gift in gifts: - if not (collection_slug := parse_collection_slug_from_gift_slug(gift.slug)): - logger.warning( - f"Can't parse collection slug from gift slug {gift.slug!r}. Skipping. " - ) - continue - entities.append( - GiftUniqueDTO.from_telethon(collection_slug=collection_slug, obj=gift) - ) - await self.telethon_service.stop() - return entities diff --git a/backend/indexer_gifts/pyproject.toml b/backend/indexer_gifts/pyproject.toml new file mode 100644 index 00000000..cbd0bbf7 --- /dev/null +++ b/backend/indexer_gifts/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = [ "setuptools==80.9.0" ] +build-backend = "setuptools.build_meta" + +[project] +name = "indexer_gifts" +version = "1.0.0" +description = "Access Community Tool – Indexer Gifts" +authors = [ + { name="danoctua", email="danoctua@gmail.com" } +] +license = { text = "MIT" } +requires-python = "==3.11.6" +dependencies = [ + "aiogram==3.24.0", +] diff --git a/backend/indexer_gifts/requirements.txt b/backend/indexer_gifts/requirements.txt index f0e34481..c2897ce5 100644 --- a/backend/indexer_gifts/requirements.txt +++ b/backend/indexer_gifts/requirements.txt @@ -2,18 +2,24 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --no-emit-index-url --output-file=backend/core/requirements.txt backend/core/pyproject.toml +# pip-compile --no-emit-index-url --output-file=backend/indexer_gifts/requirements.txt backend/core/pyproject.toml backend/indexer_gifts/pyproject.toml # aioboto3==14.1.0 # via core (backend/core/pyproject.toml) aiobotocore[boto3]==2.21.1 # via aioboto3 aiofiles==25.1.0 - # via aioboto3 + # via + # aioboto3 + # aiogram +aiogram==3.24.0 + # via indexer_gifts (backend/indexer_gifts/pyproject.toml) aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.13.2 - # via aiobotocore + # via + # aiobotocore + # aiogram aioitertools==0.13.0 # via aiobotocore aiolimiter==1.2.1 @@ -52,6 +58,7 @@ celery-types==0.23.0 # via core (backend/core/pyproject.toml) certifi==2025.10.5 # via + # aiogram # httpcore # httpx cffi==2.0.0 @@ -114,6 +121,8 @@ jmespath==1.0.1 # botocore kombu==5.6.0 # via celery +magic-filter==1.0.12 + # via aiogram mako==1.3.10 # via alembic markupsafe==3.0.3 @@ -151,6 +160,7 @@ pycparser==2.23 # via cffi pydantic==2.10.6 # via + # aiogram # core (backend/core/pyproject.toml) # fastapi # pydantic-settings @@ -196,6 +206,7 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.15.0 # via + # aiogram # aiosignal # alembic # anyio diff --git a/backend/indexer_gifts/tasks.py b/backend/indexer_gifts/tasks.py index bd400689..5c4cd6a0 100644 --- a/backend/indexer_gifts/tasks.py +++ b/backend/indexer_gifts/tasks.py @@ -1,183 +1,97 @@ import asyncio - from celery.utils.log import get_task_logger -from telethon.errors import ( - PhoneNumberBannedError, - AuthKeyDuplicatedError, - FrozenMethodInvalidError, -) from core.constants import ( - UPDATED_GIFT_USER_IDS, - CELERY_GIFT_FETCH_QUEUE_NAME, - DEFAULT_CELERY_TASK_RETRY_DELAY, + CELERY_GIFT_COLLECTIONS_SYNC_QUEUE, + CELERY_GIFT_USER_BATCH_QUEUE, + CELERY_GIFT_USER_PRIORITY_QUEUE, DEFAULT_CELERY_TASK_MAX_RETRIES, - DEFAULT_TELEGRAM_TASK_BATCH_PROCESSING_SIZE, + DEFAULT_CELERY_TASK_RETRY_DELAY, ) -from core.dtos.gift.collection import GiftCollectionDTO -from core.exceptions.gift import GiftCollectionNotExistsError from core.services.db import DBService -from core.services.gift.collection import GiftCollectionService -from core.utils.session import SessionLockManager, SessionUnavailableError -from indexer_gifts.actions.collection import IndexerGiftCollectionAction -from indexer_gifts.actions.item import IndexerGiftUniqueAction +from core.services.user import UserService +from indexer_gifts.actions.api import IndexerGiftChangesAction +from indexer_gifts.actions.user import IndexerUserGiftAction from indexer_gifts.celery_app import app -from indexer_gifts.settings import gifts_indexer_settings logger = get_task_logger(__name__) - -async def index_whitelisted_gift_collections() -> list[GiftCollectionDTO]: - """ - Asynchronously indexes whitelisted gift collections in the database. This function checks for any - gift collections that are whitelisted but missing from the database and attempts to index them. - It also ensures that the database lock related to the indexing process is released after execution. - - :return: A list of `GiftCollectionDTO` objects representing the gift collections retrieved - and indexed from the database. - """ - with DBService().db_session() as db_session: - gift_collection_service = GiftCollectionService(db_session) - collections = gift_collection_service.get_all( - slugs=gifts_indexer_settings.whitelisted_gift_collections, - ) - - collections_dtos = [GiftCollectionDTO.from_orm(c) for c in collections] - with SessionLockManager( - gifts_indexer_settings.telegram_indexer_session_path - ) as session_path: - collection_action = IndexerGiftCollectionAction( - db_session, session_path=session_path - ) - try: - for slug in gifts_indexer_settings.whitelisted_gift_collections: - try: - new_collection = await collection_action.index(slug) - collections_dtos.append(new_collection) - logger.info( - f"Whitelisted gift collection {slug!r} indexed successfully." - ) - except GiftCollectionNotExistsError as e: - logger.error(f"Failed to index gift collection {slug!r}: {e}") - except ( - PhoneNumberBannedError, - AuthKeyDuplicatedError, - FrozenMethodInvalidError, - ) as e: - # Rename session to mark as dirty - session_path.rename(f"{session_path}-dirty") - raise e - - return collections_dtos +DEFAULT_USER_GIFT_BATCH_SIZE = 25 -async def index_gift_collection_ownerships( - slug: str, start: int | None, stop: int | None -) -> None: +# === Flow A: Sync Collections === +@app.task( + name="sync-gift-collections-from-api", + queue=CELERY_GIFT_COLLECTIONS_SYNC_QUEUE, + default_retry_delay=DEFAULT_CELERY_TASK_RETRY_DELAY, + retry_kwargs={"max_retries": DEFAULT_CELERY_TASK_MAX_RETRIES}, + ignore_result=True, +) +def sync_gift_collections_from_api() -> None: """ - Indexes gift ownership data for a specified collection by processing - unique gift actions related to the given collection slug. After indexing, - updated user identifiers are logged, and certain identifiers are stored - within a Redis set for further processing. - - :param slug: The slug that uniquely identifies the gift collection whose - ownership data is being indexed. - :param start: The starting index for processing unique gift actions. - :param stop: The ending index for processing unique gift actions. + Hourly scheduled task that fetches all gift collections from the `changes.tg` API + and syncs them to the database. """ with DBService().db_session() as db_session: - with SessionLockManager( - gifts_indexer_settings.telegram_indexer_session_path - ) as session_path: - action = IndexerGiftUniqueAction(db_session, session_path=session_path) - try: - batch_telegram_ids = await action.index( - slug=slug, start=start, stop=stop - ) - logger.info( - f"Indexed {len(batch_telegram_ids)} unique gift actions for collection {slug!r}." - ) - except ( - PhoneNumberBannedError, - AuthKeyDuplicatedError, - FrozenMethodInvalidError, - ) as e: - # Rename session to mark as dirty - session_path.rename(f"{session_path}-dirty") - raise e + action = IndexerGiftChangesAction(db_session=db_session) + asyncio.run(action.index_all()) - if batch_telegram_ids: - logger.info(f"Updated user IDs count: {len(batch_telegram_ids)}") - action.redis_service.add_to_set( - UPDATED_GIFT_USER_IDS, *batch_telegram_ids - ) - logger.info(f"Gift ownerships for collection {slug!r} indexed.") +# === Flow B: Index User Gifts (High Priority Queue) === +@app.task( + name="index-user-gifts", + queue=CELERY_GIFT_USER_PRIORITY_QUEUE, + default_retry_delay=DEFAULT_CELERY_TASK_RETRY_DELAY, + retry_kwargs={"max_retries": DEFAULT_CELERY_TASK_MAX_RETRIES}, +) +def index_user_gifts(telegram_user_id: int) -> bool: + with DBService().db_session() as db_session: + action = IndexerUserGiftAction(db_session) + asyncio.run(action.sync_single_user_gifts(telegram_user_id)) + return True +# === Flow C: Recurring Background User Gift Refresh (Batch Worker) === @app.task( - name="fetch-gift-collection-ownership-details", - queue=CELERY_GIFT_FETCH_QUEUE_NAME, + name="refresh-users-gifts-batch", + queue=CELERY_GIFT_USER_BATCH_QUEUE, default_retry_delay=DEFAULT_CELERY_TASK_RETRY_DELAY, - autoretry_for=( - SessionUnavailableError, - PhoneNumberBannedError, - AuthKeyDuplicatedError, - ), retry_kwargs={"max_retries": DEFAULT_CELERY_TASK_MAX_RETRIES}, ignore_result=True, ) -def fetch_gift_collection_ownership_details( - slug: str, start: int | None = None, stop: int | None = None -) -> None: - logger.info( - f"Received task to index collection {slug!r} ownership details. Start ID: {start}, Stop ID: {stop}" - ) - asyncio.run(index_gift_collection_ownerships(slug, start=start, stop=stop)) +def refresh_users_gifts_batch(telegram_user_ids: list[int]) -> None: + """Processes a chunk of 25 users in ~30-60 seconds.""" + with DBService().db_session() as db_session: + action = IndexerUserGiftAction(db_session) + existing_collection_ids = action.gift_collection_service.get_all_ids() + + asyncio.run(action.sync_users_batch(telegram_user_ids, existing_collection_ids)) +# === Flow C: Master Dispatcher Task === @app.task( - name="fetch-gift-ownership-details", - queue=CELERY_GIFT_FETCH_QUEUE_NAME, - default_retry_delay=DEFAULT_CELERY_TASK_RETRY_DELAY, - autoretry_for=( - SessionUnavailableError, - PhoneNumberBannedError, - AuthKeyDuplicatedError, - ), - retry_kwargs={"max_retries": DEFAULT_CELERY_TASK_MAX_RETRIES}, + name="refresh-all-user-gifts", + queue=CELERY_GIFT_USER_BATCH_QUEUE, ignore_result=True, ) -def fetch_gift_ownership_details(): - """ - Fetch details of gift ownership for each gift collection in a whitelisted list. +def refresh_all_user_gifts() -> None: + """Master dispatcher: chunks all registered users and enqueues batch tasks.""" + with DBService().db_session() as db_session: + user_service = UserService(db_session) + all_telegram_ids = user_service.get_all_telegram_ids() - This task fetches ownership details for gift collections by dividing - the processing into batches of a predefined size. - It uses the Celery task queue system and works in conjunction with the function that fetches - ownership details for each batch. + if not all_telegram_ids: + logger.info("No registered Telegram users found for gift refresh.") + return - The function runs asynchronously to index whitelisted gift collections - and iterates through these collections to process their ownership details. - It dispatches separate tasks for each batch of upgraded items within the - collections. - Tasks will be retried automatically on session or phone-number-related errors - up to a maximum defined limit. - """ - collections = asyncio.run(index_whitelisted_gift_collections()) - for collection in collections: - for i in range( - 1, collection.upgraded_count, DEFAULT_TELEGRAM_TASK_BATCH_PROCESSING_SIZE - ): - app.send_task( - "fetch-gift-collection-ownership-details", - args=( - collection.slug, - i, - min( - i + DEFAULT_TELEGRAM_TASK_BATCH_PROCESSING_SIZE, - collection.upgraded_count, - ), - ), - queue=CELERY_GIFT_FETCH_QUEUE_NAME, - ) + logger.info( + f"Dispatching gift refresh for {len(all_telegram_ids)} users in chunks of {DEFAULT_USER_GIFT_BATCH_SIZE}" + ) + + for i in range(0, len(all_telegram_ids), DEFAULT_USER_GIFT_BATCH_SIZE): + chunk = all_telegram_ids[i : i + DEFAULT_USER_GIFT_BATCH_SIZE] + app.send_task( + "refresh-users-gifts-batch", + args=(chunk,), + queue=CELERY_GIFT_USER_BATCH_QUEUE, + ) diff --git a/backend/indexer_price/actions/__init__.py b/backend/indexer_price/actions/__init__.py index c9154a82..b7a94c7a 100644 --- a/backend/indexer_price/actions/__init__.py +++ b/backend/indexer_price/actions/__init__.py @@ -44,9 +44,9 @@ def __init__(self, db_session: Session) -> None: async def refresh_toncoin_price(self) -> None: try: new_price = await self.indexer.index() - logger.info(f"Successfully refreshed TON price. New price: {new_price}") + logger.info(f"Successfully refreshed GRAM price. New price: {new_price}") except Exception as e: - logger.exception(f"Error occurred while refreshing TON price: {e}") + logger.exception(f"Error occurred while refreshing GRAM price: {e}") class JettonPriceIndexerAction(PriceIndexerAction): @@ -150,7 +150,7 @@ def _validate_nft_collection_status( async def refresh_nft_collections_price(self) -> None: if not (ton_price := self.ton_price_manager.get_ton_price()): logger.warning( - "Cannot refresh NFT collections prices. TON price is not set" + "Cannot refresh NFT collections prices. GRAM price is not set" ) return diff --git a/backend/indexer_price/indexers/ton.py b/backend/indexer_price/indexers/ton.py index ab064aa2..2adc370b 100644 --- a/backend/indexer_price/indexers/ton.py +++ b/backend/indexer_price/indexers/ton.py @@ -17,17 +17,17 @@ def __init__(self) -> None: async def index(self) -> float: """ - Fetches the latest price for the TON asset from the given API endpoint and sets + Fetches the latest price for the GRAM asset from the given API endpoint and sets the fetched price. This method performs an asynchronous GET request to obtain the current asset - quotation for TON and validates the response. Upon success, it updates the - price of TON and logs the operation. + quotation for GRAM and validates the response. Upon success, it updates the + price of GRAM and logs the operation. :raises HTTPError: If the HTTP request fails or returns an unsuccessful response code. :raises ValidationError: If the response cannot be validated against the TonPrice model. - :return: The latest price of the TON asset. + :return: The latest price of the GRAM asset. :rtype: float """ response = await self.client.get( @@ -35,6 +35,6 @@ async def index(self) -> float: ) response.raise_for_status() ton_data = TonPrice.model_validate(response.json()) - logger.info(f"Got new price for TON: {ton_data.price}") + logger.info(f"Got new price for GRAM: {ton_data.price}") self.price_manager.set_ton_price(ton_data.price) return ton_data.price diff --git a/backend/indexer_price/tasks.py b/backend/indexer_price/tasks.py index 6b52edb6..6915bfc0 100644 --- a/backend/indexer_price/tasks.py +++ b/backend/indexer_price/tasks.py @@ -20,9 +20,9 @@ async def refresh_toncoin_price(): with DBService().db_session() as db_session: action = TonPriceIndexerAction(db_session=db_session) - logger.info("Started TON prices refreshing action") + logger.info("Started GRAM prices refreshing action") await action.refresh_toncoin_price() - logger.info("Successfully completed TON prices refreshing action") + logger.info("Successfully completed GRAM prices refreshing action") async def refresh_jettons_price(): diff --git a/backend/scheduler/celery_app.py b/backend/scheduler/celery_app.py index 5b9b06b7..99c4c601 100644 --- a/backend/scheduler/celery_app.py +++ b/backend/scheduler/celery_app.py @@ -5,8 +5,9 @@ CELERY_NOTICED_WALLETS_UPLOAD_QUEUE_NAME, CELERY_SYSTEM_QUEUE_NAME, CELERY_STICKER_FETCH_QUEUE_NAME, - CELERY_GIFT_FETCH_QUEUE_NAME, CELERY_INDEX_PRICES_QUEUE_NAME, + CELERY_GIFT_COLLECTIONS_SYNC_QUEUE, + CELERY_GIFT_USER_BATCH_QUEUE, ) from core.settings import core_settings @@ -48,10 +49,20 @@ def create_app() -> Celery: "schedule": crontab(minute="*/10"), # Every 10 minutes "options": {"queue": CELERY_STICKER_FETCH_QUEUE_NAME}, }, - "fetch-gift-ownerships": { - "task": "fetch-gift-ownership-details", - "schedule": crontab(hour="*/1", minute="0"), # Every hour - "options": {"queue": CELERY_GIFT_FETCH_QUEUE_NAME}, + # "fetch-gift-ownerships": { + # "task": "fetch-gift-ownership-details", + # "schedule": crontab(hour="*/1", minute="0"), # Every hour + # "options": {"queue": CELERY_GIFT_COLLECTIONS_SYNC_QUEUE}, + # }, + "sync-gift-collections-from-api": { + "task": "sync-gift-collections-from-api", + "schedule": crontab(hour="1", minute="0"), # Daily at 1:00 AM + "options": {"queue": CELERY_GIFT_COLLECTIONS_SYNC_QUEUE}, + }, + "refresh-all-user-gifts": { + "task": "refresh-all-user-gifts", + "schedule": crontab(hour="*/1", minute="15"), # Hourly at :15 + "options": {"queue": CELERY_GIFT_USER_BATCH_QUEUE}, }, "refresh-prices": { "task": "refresh-prices", diff --git a/backend/tests/unit/community_manager/actions/test_chat_kick.py b/backend/tests/unit/community_manager/actions/test_chat_kick.py index 32ac8f31..d669b899 100644 --- a/backend/tests/unit/community_manager/actions/test_chat_kick.py +++ b/backend/tests/unit/community_manager/actions/test_chat_kick.py @@ -1,18 +1,27 @@ import pytest +import logging from unittest.mock import AsyncMock, MagicMock, patch from community_manager.actions.chat import CommunityManagerUserChatAction -from core.models.chat import TelegramChatUser, TelegramChat -from core.models.user import User +from core.dtos.chat.rule.internal import ( + ChatMemberEligibilityResultDTO, + EligibilitySummaryInternalDTO, + RulesEligibilityGroupSummaryInternalDTO, + RulesEligibilitySummaryInternalDTO, +) +from core.enums.rule import EligibilityCheckType +from tests.factories import TelegramChatFactory, TelegramChatUserFactory, UserFactory @pytest.mark.asyncio async def test_kick_chat_member_admin_protection(db_session): action = CommunityManagerUserChatAction(db_session) - chat = TelegramChat(id=1, title="Test Chat", is_full_control=True) - user = User(id=1, telegram_id=123) - chat_user = TelegramChatUser( - user_id=1, chat_id=1, is_admin=True, is_managed=True, chat=chat, user=user + chat = TelegramChatFactory.with_session(db_session).create( + id=1, title="Test Chat", is_full_control=True + ) + user = UserFactory.with_session(db_session).create(id=1, telegram_id=123) + chat_user = TelegramChatUserFactory.with_session(db_session).create( + user=user, chat=chat, is_admin=True, is_managed=True ) # Mock bot_api_service to ensure it is NOT called @@ -26,10 +35,12 @@ async def test_kick_chat_member_admin_protection(db_session): @pytest.mark.asyncio async def test_kick_chat_member_normal_user(db_session): action = CommunityManagerUserChatAction(db_session) - chat = TelegramChat(id=1, title="Test Chat", is_full_control=True) - user = User(id=1, telegram_id=123) - chat_user = TelegramChatUser( - user_id=1, chat_id=1, is_admin=False, is_managed=True, chat=chat, user=user + chat = TelegramChatFactory.with_session(db_session).create( + id=1, title="Test Chat", is_full_control=True + ) + user = UserFactory.with_session(db_session).create(id=1, telegram_id=123) + chat_user = TelegramChatUserFactory.with_session(db_session).create( + user=user, chat=chat, is_admin=False, is_managed=True ) # Mock bot_api_service context manager @@ -45,3 +56,144 @@ async def test_kick_chat_member_normal_user(db_session): await action.kick_chat_member(chat_user) mock_service_instance.kick_chat_member.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_check_chat_members_compliance_dry_run_counters(db_session, caplog): + action = CommunityManagerUserChatAction(db_session) + chat = TelegramChatFactory.with_session(db_session).create( + id=1, title="Test Chat", is_full_control=True + ) + + # Create 3 users: + # 1. Managed and ineligible + # 2. Non-managed and ineligible + # 3. Non-managed and eligible (to test total non_managed counting) + user1 = UserFactory.with_session(db_session).create(id=1, telegram_id=111) + chat_user1 = TelegramChatUserFactory.with_session(db_session).create( + user=user1, chat=chat, is_admin=False, is_managed=True + ) + + user2 = UserFactory.with_session(db_session).create(id=2, telegram_id=222) + chat_user2 = TelegramChatUserFactory.with_session(db_session).create( + user=user2, chat=chat, is_admin=False, is_managed=False + ) + + user3 = UserFactory.with_session(db_session).create(id=3, telegram_id=333) + chat_user3 = TelegramChatUserFactory.with_session(db_session).create( + user=user3, chat=chat, is_admin=False, is_managed=False + ) + + # Mock telegram_chat_user_service yield + action.telegram_chat_user_service = MagicMock() + action.telegram_chat_user_service.yield_all_for_chat.return_value = [ + [chat_user1, chat_user2, chat_user3] + ] + + # Mock evaluate_chat_members_eligibility + action.authorization_action = MagicMock() + + res1 = ChatMemberEligibilityResultDTO( + member=chat_user1, + is_eligible=False, + summary=RulesEligibilitySummaryInternalDTO( + groups=[ + RulesEligibilityGroupSummaryInternalDTO( + id=1, + items=[ + EligibilitySummaryInternalDTO( + id=1, + group_id=1, + type=EligibilityCheckType.TONCOIN, + title="GRAM", + actual=0.5, + expected=1.0, + is_enabled=True, + ) + ], + ) + ], + wallet="EQD123", + ), + ) + res2 = ChatMemberEligibilityResultDTO( + member=chat_user2, + is_eligible=False, + summary=RulesEligibilitySummaryInternalDTO( + groups=[ + RulesEligibilityGroupSummaryInternalDTO( + id=2, + items=[ + EligibilitySummaryInternalDTO( + id=2, + group_id=2, + type=EligibilityCheckType.JETTON, + title="USDT", + actual=0, + expected=100, + is_enabled=True, + ) + ], + ) + ], + wallet=None, + ), + ) + res3 = ChatMemberEligibilityResultDTO( + member=chat_user3, + is_eligible=True, + summary=RulesEligibilitySummaryInternalDTO(groups=[]), + ) + + action.authorization_action.evaluate_chat_members_eligibility.return_value = [ + res1, + res2, + res3, + ] + + with caplog.at_level(logging.INFO): + await action.check_chat_members_compliance_dry_run(chat_id=1) + + # Check that counts were correctly logged + # Total processed: 3, Non-managed: 2, Ineligible (managed): 1, Ineligible (non-managed): 1 + assert "Total processed: 3" in caplog.text + assert "Non-managed: 2" in caplog.text + assert "Ineligible (managed): 1" in caplog.text + assert "Ineligible (non-managed): 1" in caplog.text + + assert ( + 'User 111 is ineligible for chat 1. Managed: True. Compliance summary: {"groups":[{"id":1,"items":[{"id":1,"group_id":1,"type":"toncoin","title":"GRAM","address_raw":null,"actual":0.5,"expected":1.0,"is_enabled":true,"category":null,"is_eligible":false}]}],"wallet":"EQD123"}' + in caplog.text + ) + assert ( + 'User 222 is ineligible for chat 1. Managed: False. Compliance summary: {"groups":[{"id":2,"items":[{"id":2,"group_id":2,"type":"jetton","title":"USDT","address_raw":null,"actual":0,"expected":100,"is_enabled":true,"category":null,"is_eligible":false}]}],"wallet":null}' + in caplog.text + ) + + +@pytest.mark.asyncio +async def test_kick_ineligible_chat_members_sets_managed_flag(db_session): + action = CommunityManagerUserChatAction(db_session) + chat = TelegramChatFactory.with_session(db_session).create( + id=1, title="Test Chat", is_full_control=True + ) + + user1 = UserFactory.with_session(db_session).create(id=1, telegram_id=111) + chat_user1 = TelegramChatUserFactory.with_session(db_session).create( + user=user1, chat=chat, is_admin=False, is_managed=False + ) + + action.authorization_action = MagicMock() + + res1 = ChatMemberEligibilityResultDTO( + member=chat_user1, + is_eligible=True, + summary=RulesEligibilitySummaryInternalDTO(groups=[]), + ) + action.authorization_action.evaluate_chat_members_eligibility.return_value = [res1] + + assert chat_user1.is_managed is False + + await action.kick_ineligible_chat_members([chat_user1]) + + assert chat_user1.is_managed is True diff --git a/backend/tests/unit/core/actions/test_user.py b/backend/tests/unit/core/actions/test_user.py new file mode 100644 index 00000000..ea3c41f5 --- /dev/null +++ b/backend/tests/unit/core/actions/test_user.py @@ -0,0 +1,71 @@ +from unittest.mock import patch, MagicMock + +from core.actions.user import UserAction +from core.constants import ( + CELERY_GIFT_USER_PRIORITY_QUEUE, + USER_GIFT_REFRESH_COOLDOWN_SECONDS, +) +from tests.factories.user import UserFactory + + +def test_refresh_gifts_success(db_session): + with patch("core.actions.user.sender.send_task") as mock_send_task, patch( + "core.actions.user.RedisService" + ) as mock_redis_class: + # Mock Redis to allow the request + mock_redis = MagicMock() + mock_redis.set.return_value = True + mock_redis_class.return_value = mock_redis + + mock_task_result = MagicMock() + mock_task_result.id = "mock-task-123" + mock_send_task.return_value = mock_task_result + + user_action = UserAction(db_session) + user = UserFactory.with_session(db_session).create(telegram_id=555) + db_session.flush() + + task_id = user_action.refresh_gifts(user) + + assert task_id == "mock-task-123" + mock_redis.set.assert_called_once_with( + "user-gift-refresh-555", "1", ex=USER_GIFT_REFRESH_COOLDOWN_SECONDS, nx=True + ) + mock_send_task.assert_called_once_with( + "index-user-gifts", + args=(555,), + queue=CELERY_GIFT_USER_PRIORITY_QUEUE, + ) + + +def test_refresh_gifts_rate_limited(db_session): + with patch("core.actions.user.sender.send_task") as mock_send_task, patch( + "core.actions.user.RedisService" + ) as mock_redis_class: + # Mock Redis to block the request (nx=True returns False) + mock_redis = MagicMock() + mock_redis.set.return_value = False + mock_redis_class.return_value = mock_redis + + user_action = UserAction(db_session) + user = UserFactory.with_session(db_session).create(telegram_id=555) + db_session.flush() + + task_id = user_action.refresh_gifts(user) + + assert task_id is None + mock_redis.set.assert_called_once_with( + "user-gift-refresh-555", "1", ex=USER_GIFT_REFRESH_COOLDOWN_SECONDS, nx=True + ) + mock_send_task.assert_not_called() + + +def test_initial_user_indexing(db_session): + with patch.object(UserAction, "refresh_gifts") as mock_refresh_gifts: + user_action = UserAction(db_session) + user = UserFactory.with_session(db_session).create(telegram_id=777) + db_session.flush() + + user_action._initial_user_indexing(user) + + mock_refresh_gifts.assert_called_once_with(user) diff --git a/backend/tests/unit/core/services/test_user.py b/backend/tests/unit/core/services/test_user.py new file mode 100644 index 00000000..04856a90 --- /dev/null +++ b/backend/tests/unit/core/services/test_user.py @@ -0,0 +1,20 @@ +from core.services.user import UserService +from tests.factories.user import UserFactory + + +def test_get_all_telegram_ids(db_session): + user_service = UserService(db_session) + + # Create some users with and without telegram_ids + UserFactory.with_session(db_session).create(telegram_id=10001) + UserFactory.with_session(db_session).create(telegram_id=10002) + UserFactory.with_session(db_session).create(telegram_id=None) + + db_session.flush() + + telegram_ids = user_service.get_all_telegram_ids() + assert 10001 in telegram_ids + assert 10002 in telegram_ids + assert None not in telegram_ids + + assert set([10001, 10002]).issubset(set(telegram_ids)) diff --git a/docker-compose.yml b/docker-compose.yml index 55ad7e5c..097974c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,7 +114,7 @@ services: "indexer_gifts.celery_app:app", "worker", "-Q", - "gift-fetch-queue", + "gift-user-priority,gift-collections-sync,gift-user-batch", "--loglevel=info" ] environment: diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index 0d5f9780..9df48049 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -18,7 +18,7 @@ The frontend is a single-page application (SPA) built with React and Vite. - **State Management**: Zustand (Global state), TanStack Query (Server state). - **Routing**: React Router DOM. - **UI Libraries**: - - `@tonconnect/ui-react`: TON wallet connection. + - `@tonconnect/ui-react`: GRAM wallet connection. - `@dnd-kit`: Drag and drop functionality. - `react-lottie`: Lottie animations. diff --git a/frontend/src/components/ConditionIcon/ConditionIcon.tsx b/frontend/src/components/ConditionIcon/ConditionIcon.tsx index d02caca8..3e64f1df 100644 --- a/frontend/src/components/ConditionIcon/ConditionIcon.tsx +++ b/frontend/src/components/ConditionIcon/ConditionIcon.tsx @@ -12,7 +12,7 @@ interface ConditionIconProps { } const ICONS = { - toncoin: 'toncoin', + toncoin: 'gram', 'connect-wallet': 'connectWallet', premium: 'premium', whitelist: 'whitelist', diff --git a/frontend/src/components/Icon/helpers.ts b/frontend/src/components/Icon/helpers.ts index c2684127..b32eab46 100644 --- a/frontend/src/components/Icon/helpers.ts +++ b/frontend/src/components/Icon/helpers.ts @@ -11,7 +11,7 @@ import { eyeCrossed, eye, connectWallet, - toncoin, + gram, premium, whitelist, externalSource, @@ -24,6 +24,7 @@ import { completed, sortArrows, checkmark, + refresh, } from './icons' import { IconTypeName } from './types' @@ -53,8 +54,8 @@ export const getIcon = (name: IconTypeName) => { return eye case 'connectWallet': return connectWallet - case 'toncoin': - return toncoin + case 'gram': + return gram case 'premium': return premium case 'whitelist': @@ -79,5 +80,7 @@ export const getIcon = (name: IconTypeName) => { return sortArrows case 'checkmark': return checkmark + case 'refresh': + return refresh } } diff --git a/frontend/src/components/Icon/icons/conditions/gram.tsx b/frontend/src/components/Icon/icons/conditions/gram.tsx new file mode 100644 index 00000000..20342146 --- /dev/null +++ b/frontend/src/components/Icon/icons/conditions/gram.tsx @@ -0,0 +1,39 @@ +export const gram = ( + + + + + + + + + + + + + + + +) diff --git a/frontend/src/components/Icon/icons/conditions/index.ts b/frontend/src/components/Icon/icons/conditions/index.ts index a7cbff12..29b72af4 100644 --- a/frontend/src/components/Icon/icons/conditions/index.ts +++ b/frontend/src/components/Icon/icons/conditions/index.ts @@ -1,5 +1,5 @@ export * from './connectWallet' -export * from './toncoin' +export * from './gram' export * from './premium' export * from './whitelist' export * from './externalSource' diff --git a/frontend/src/components/Icon/icons/conditions/toncoin.tsx b/frontend/src/components/Icon/icons/conditions/toncoin.tsx deleted file mode 100644 index 84800dbb..00000000 --- a/frontend/src/components/Icon/icons/conditions/toncoin.tsx +++ /dev/null @@ -1,30 +0,0 @@ -export const toncoin = ( - - - - - - - - - - - -) diff --git a/frontend/src/components/Icon/icons/index.ts b/frontend/src/components/Icon/icons/index.ts index 1009498a..0208b0a6 100644 --- a/frontend/src/components/Icon/icons/index.ts +++ b/frontend/src/components/Icon/icons/index.ts @@ -14,3 +14,4 @@ export * from './checkUnable' export * from './completed' export * from './sortArrows' export * from './checkmark' +export * from './refresh' diff --git a/frontend/src/components/Icon/icons/refresh.tsx b/frontend/src/components/Icon/icons/refresh.tsx new file mode 100644 index 00000000..23b08053 --- /dev/null +++ b/frontend/src/components/Icon/icons/refresh.tsx @@ -0,0 +1,11 @@ +export const refresh = ( + + + +) diff --git a/frontend/src/components/Icon/types.ts b/frontend/src/components/Icon/types.ts index 830d49b1..3ed4e084 100644 --- a/frontend/src/components/Icon/types.ts +++ b/frontend/src/components/Icon/types.ts @@ -5,7 +5,7 @@ export type IconTypeName = | 'pixel' | 'dogs' | 'notcoin' - | 'toncoin' + | 'gram' | 'forwardArrow' | 'gatewayBot' | 'doubleChevron' @@ -17,7 +17,7 @@ export type IconTypeName = | 'eyeCrossed' | 'eye' | 'connectWallet' - | 'toncoin' + | 'gram' | 'premium' | 'whitelist' | 'externalSource' @@ -30,5 +30,6 @@ export type IconTypeName = | 'completed' | 'sortArrows' | 'checkmark' + | 'refresh' export type IconSize = 112 | 100 | 40 | 28 | 24 | 20 | 18 | 16 | 12 | 7 diff --git a/frontend/src/pages/admin/ConditionPage/components/Gifts/Gifts.tsx b/frontend/src/pages/admin/ConditionPage/components/Gifts/Gifts.tsx index 2cd79183..ea5062c5 100644 --- a/frontend/src/pages/admin/ConditionPage/components/Gifts/Gifts.tsx +++ b/frontend/src/pages/admin/ConditionPage/components/Gifts/Gifts.tsx @@ -1,14 +1,14 @@ -import { AppSelect, Image, ListInput, ListItem, Text } from '@components' -import { List } from '@components' -import { Block } from '@components' -import { useEffect, useState } from 'react' +import { AppSelect, Image, ListInput, ListItem, Text } from '@components'; +import { List } from '@components'; +import { Block } from '@components'; +import { useEffect } from 'react'; -import { Condition, GiftsCollection } from '@store' -import { useCondition } from '@store' -import { useConditionActions } from '@store' +import { Condition, GiftsCollection } from '@store'; +import { useCondition } from '@store'; +import { useConditionActions } from '@store'; -import { ConditionComponentProps } from '../types' -import { Skeleton } from './Skeleton' +import { ConditionComponentProps } from '../types'; +import { Skeleton } from './Skeleton'; export const Gifts = ({ isNewCondition, @@ -21,9 +21,6 @@ export const Gifts = ({ useConditionActions() const { giftsData } = useCondition() - const [currentCollection, setCurrentCollection] = - useState(null) - const fetchGifts = async () => { try { await fetchGiftsAction() @@ -39,31 +36,23 @@ export const Gifts = ({ useEffect(() => { if (giftsData?.length) { + const selectedCollectionId: string = + condition?.collectionId != null + ? String(condition.collectionId) + : (condition?.collection as GiftsCollection)?.id != null + ? String((condition?.collection as GiftsCollection).id) + : giftsData[0].id + const updatedConditionState: Partial = { type: 'gift_collection', category: null, - isEnabled: !!condition?.isEnabled || true, - collectionSlug: - condition?.collectionSlug || - condition?.slug || - (condition?.collection as GiftsCollection)?.slug || - giftsData[0].slug, - model: condition?.model || null, - backdrop: condition?.backdrop || null, - pattern: condition?.pattern || null, - expected: condition?.expected || '', + isEnabled: condition?.isEnabled ?? true, + collectionId: selectedCollectionId, + model: condition?.model ?? null, + backdrop: condition?.backdrop ?? null, + pattern: condition?.pattern ?? null, + expected: condition?.expected ?? '', } - - setCurrentCollection( - giftsData.find( - (collection) => - collection.slug === - (condition?.collectionSlug || - condition?.slug || - (condition?.collection as GiftsCollection)?.slug) - ) || giftsData[0] - ) - setInitialState(updatedConditionState as Partial) } }, [giftsData?.length, condition, isNewCondition]) @@ -72,6 +61,12 @@ export const Gifts = ({ return } + // derive currentCollection from the authoritative conditionState.collectionId + const currentCollection: GiftsCollection = + giftsData.find((collection) => + collection.id === String(conditionState?.collectionId ?? giftsData[0].id) + ) || giftsData[0] + return ( <> @@ -81,18 +76,22 @@ export const Gifts = ({ after={ { - setCurrentCollection( - giftsData.find((collection) => collection.slug === value) || - giftsData[0] - ) + const selectedIdStr = String(value) + + // reset dependent fields handleChangeCondition('model', null) handleChangeCondition('backdrop', null) handleChangeCondition('pattern', null) - handleChangeCondition('collectionSlug', value) + + // store selected collection id in condition state + handleChangeCondition('collectionId', selectedIdStr) }} - value={conditionState?.collectionSlug} + // the select value should reflect the selected collection id (string) + value={String( + conditionState?.collectionId ?? currentCollection?.id ?? giftsData[0].id + )} options={giftsData.map((collection) => ({ - value: collection.slug, + value: collection.id, name: collection.title, }))} /> @@ -135,7 +134,7 @@ export const Gifts = ({ value: 'Any', name: 'Any', }, - ...(currentCollection?.models || []).map((model) => ({ + ...(currentCollection?.options?.models || []).map((model) => ({ value: model, name: model, })), @@ -160,7 +159,7 @@ export const Gifts = ({ value: 'Any', name: 'Any', }, - ...(currentCollection?.backdrops || []).map((backdrop) => ({ + ...(currentCollection?.options?.backdrops || []).map((backdrop) => ({ value: backdrop, name: backdrop, })), @@ -185,7 +184,7 @@ export const Gifts = ({ value: 'Any', name: 'Any', }, - ...(currentCollection?.patterns || []).map((pattern) => ({ + ...(currentCollection?.options?.patterns || []).map((pattern) => ({ value: pattern, name: pattern, })), diff --git a/frontend/src/pages/admin/ConditionPage/components/Toncoin/Toncoin.tsx b/frontend/src/pages/admin/ConditionPage/components/Gram/Gram.tsx similarity index 98% rename from frontend/src/pages/admin/ConditionPage/components/Toncoin/Toncoin.tsx rename to frontend/src/pages/admin/ConditionPage/components/Gram/Gram.tsx index 3a839425..044a1896 100644 --- a/frontend/src/pages/admin/ConditionPage/components/Toncoin/Toncoin.tsx +++ b/frontend/src/pages/admin/ConditionPage/components/Gram/Gram.tsx @@ -5,7 +5,7 @@ import { useConditionActions, ConditionCategory, Condition } from '@store' import { ConditionComponentProps } from '../types' -export const Toncoin = ({ +export const Gram = ({ isNewCondition, handleChangeCondition, conditionState, @@ -90,7 +90,7 @@ export const Toncoin = ({ textColor="tertiary" after={ - TON + GRAM } value={conditionState?.expected} diff --git a/frontend/src/pages/admin/ConditionPage/components/Toncoin/Skeleton.tsx b/frontend/src/pages/admin/ConditionPage/components/Gram/Skeleton.tsx similarity index 100% rename from frontend/src/pages/admin/ConditionPage/components/Toncoin/Skeleton.tsx rename to frontend/src/pages/admin/ConditionPage/components/Gram/Skeleton.tsx diff --git a/frontend/src/pages/admin/ConditionPage/components/Gram/index.ts b/frontend/src/pages/admin/ConditionPage/components/Gram/index.ts new file mode 100644 index 00000000..e7aa6ef9 --- /dev/null +++ b/frontend/src/pages/admin/ConditionPage/components/Gram/index.ts @@ -0,0 +1 @@ +export * from './Gram' diff --git a/frontend/src/pages/admin/ConditionPage/components/Toncoin/index.ts b/frontend/src/pages/admin/ConditionPage/components/Toncoin/index.ts deleted file mode 100644 index 1c96c43f..00000000 --- a/frontend/src/pages/admin/ConditionPage/components/Toncoin/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './Toncoin' diff --git a/frontend/src/pages/admin/ConditionPage/components/index.ts b/frontend/src/pages/admin/ConditionPage/components/index.ts index 5321a926..6e1d6d44 100644 --- a/frontend/src/pages/admin/ConditionPage/components/index.ts +++ b/frontend/src/pages/admin/ConditionPage/components/index.ts @@ -1,6 +1,6 @@ export * from './Jettons' export * from './NFT' -export * from './Toncoin' +export * from './Gram' export * from './Whitelist' export * from './Premium' export * from './Emoji' diff --git a/frontend/src/pages/admin/ConditionPage/constants.ts b/frontend/src/pages/admin/ConditionPage/constants.ts index 2e6d88f0..45acb1fb 100644 --- a/frontend/src/pages/admin/ConditionPage/constants.ts +++ b/frontend/src/pages/admin/ConditionPage/constants.ts @@ -3,7 +3,7 @@ import { Jettons, NFT, Premium, - Toncoin, + Gram, Whitelist, ExternalSource, Stickers, @@ -17,7 +17,7 @@ export const CONDITION_TYPES = [ }, { value: 'toncoin', - name: 'Toncoin', + name: 'Gram', }, { value: 'nft_collection', @@ -53,7 +53,7 @@ export const CONDITION_COMPONENTS = { jetton: Jettons, nft_collection: NFT, whitelist: Whitelist, - toncoin: Toncoin, + toncoin: Gram, premium: Premium, emoji: Emoji, external_source: ExternalSource, diff --git a/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/ChatConditions.tsx b/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/ChatConditions.tsx index 7b1a702d..c1b4da39 100644 --- a/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/ChatConditions.tsx +++ b/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/ChatConditions.tsx @@ -1,6 +1,7 @@ -import { Block, List } from '@components' +import { useState } from 'react' +import { Block, List, useToast } from '@components' -import { useChat } from '@store' +import { useChat, useUserActions, useChatActions } from '@store' import { checkWalletRequirements } from '../../helpers' import { ChatConditionItem, WalletCondition } from './components' @@ -9,9 +10,32 @@ const SEPARATOR_LEFT_GAP = 40 export const ChatConditions = () => { const { chat, groups, rules } = useChat() + const { refreshUserGiftsAction, completeChatTaskAction } = useUserActions() + const { fetchUserChatAction } = useChatActions() + const { showToast } = useToast() + + const [isRefreshingGifts, setIsRefreshingGifts] = useState(false) const renderWallet = checkWalletRequirements(rules) + const handleRefreshGifts = async () => { + setIsRefreshingGifts(true) + try { + const taskId = await refreshUserGiftsAction() + if (taskId) { + await completeChatTaskAction(taskId) + } + if (chat?.slug) { + await fetchUserChatAction(chat.slug) + } + } catch (e: unknown) { + const message = e instanceof Error ? e.message : 'Failed to refresh gifts' + showToast({ type: 'error', message }) + } finally { + setIsRefreshingGifts(false) + } + } + return ( <> {groups?.map((group, index) => { @@ -24,6 +48,9 @@ export const ChatConditions = () => { condition={condition} key={`${condition.id}-${condition.type}`} chat={chat} + disabled={isRefreshingGifts} + onRefreshGifts={handleRefreshGifts} + isRefreshingGifts={isRefreshingGifts} /> ))} diff --git a/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/components/ChatConditionItem/ChatConditionItem.tsx b/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/components/ChatConditionItem/ChatConditionItem.tsx index cf07981f..9998ff88 100644 --- a/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/components/ChatConditionItem/ChatConditionItem.tsx +++ b/frontend/src/pages/client/ClientTasksPage/components/ChatConditions/components/ChatConditionItem/ChatConditionItem.tsx @@ -1,4 +1,4 @@ -import { ConditionIcon, ListItem, Text } from '@components' +import { ConditionIcon, ListItem, Text, Icon, Spinner } from '@components' import { createConditionName, goTo } from '@utils' import { createConditionDescription } from '@utils' @@ -10,12 +10,16 @@ interface ChatConditionItemProps { condition: Condition chat: ChatInstance | null disabled?: boolean + onRefreshGifts?: () => void + isRefreshingGifts?: boolean } export const ChatConditionItem = ({ condition, chat, disabled, + onRefreshGifts, + isRefreshingGifts, }: ChatConditionItemProps) => { const { isEligible, promoteUrl, type } = condition @@ -85,14 +89,25 @@ export const ChatConditionItem = ({ ) } + const isGiftCondition = type === 'gift_collection' + return ( } disabled={disabled} + after={ + isGiftCondition ? ( + isRefreshingGifts ? ( + + ) : ( + + ) + ) : undefined + } text={{conditionName}} description={ diff --git a/frontend/src/store/condition/types.ts b/frontend/src/store/condition/types.ts index 7b55cdc1..c94f0ddf 100644 --- a/frontend/src/store/condition/types.ts +++ b/frontend/src/store/condition/types.ts @@ -102,12 +102,14 @@ export interface StickersCollection { } export interface GiftsCollection { + id: string, title: string - backdrops: string[] - models: string[] - patterns: string[] + options: { + backdrops: string[] + models: string[] + patterns: string[] + } previewUrl: string - slug: string supply: number upgradedCount: number } diff --git a/frontend/src/store/user/api.ts b/frontend/src/store/user/api.ts index f5c1d1ad..cbce76d0 100644 --- a/frontend/src/store/user/api.ts +++ b/frontend/src/store/user/api.ts @@ -72,3 +72,11 @@ export const completeChatTasksAPI = async ( return response } + +export const refreshUserGiftsAPI = async (): Promise> => { + const response = await ApiService.post<{ status: string, taskId?: string, message: string }>({ + endpoint: '/users/me/gifts/refresh', + }) + + return response +} diff --git a/frontend/src/store/user/user.ts b/frontend/src/store/user/user.ts index 2bafb318..25ca8efd 100644 --- a/frontend/src/store/user/user.ts +++ b/frontend/src/store/user/user.ts @@ -10,6 +10,7 @@ import { connectExistingWalletAPI, connectWalletAPI, fetchUserAPI, + refreshUserGiftsAPI, } from './api' import { User, WalletData } from './types' @@ -31,6 +32,7 @@ interface UserActions { wallet: string ) => Promise completeChatTaskAction: (taskId: string) => void + refreshUserGiftsAction: () => Promise } } @@ -95,6 +97,13 @@ const useUserStore = create((set) => ({ throw new Error(error) } }, + refreshUserGiftsAction: async () => { + const { data, ok, error } = await refreshUserGiftsAPI() + if (!ok) { + throw new Error(error) + } + return data?.taskId + }, }, })) diff --git a/specs/telegram-gifts-indexer/contracts/api.md b/specs/telegram-gifts-indexer/contracts/api.md new file mode 100644 index 00000000..670f18ad --- /dev/null +++ b/specs/telegram-gifts-indexer/contracts/api.md @@ -0,0 +1,43 @@ +# API Contracts: Telegram Bot API Gift Indexing + +## REST API (FastAPI) + +### POST `/api/users/me/gifts/refresh` + +Triggers a manual refresh of the authenticated user's Telegram unique gifts. + +**Headers**: +- `Authorization`: `Bearer ` + +**Response: Success (200 OK)** +```json +{ + "status": "pending", + "task_id": "uuid-1234-5678", + "message": "Gift indexing started." +} +``` + +**Response: Rate Limited (200 OK)** +```json +{ + "status": "rate_limited", + "task_id": null, + "message": "Gifts were recently indexed. Please wait 5 minutes." +} +``` + +**Response: Unauthorized (401 Unauthorized)** +```json +{ + "detail": "Could not validate credentials" +} +``` + +**Response: Bad Request (400 Bad Request)** +*(If the user has no linked Telegram ID)* +```json +{ + "detail": "User has no linked Telegram ID" +} +``` diff --git a/specs/telegram-gifts-indexer/data-model.md b/specs/telegram-gifts-indexer/data-model.md new file mode 100644 index 00000000..32b636d8 --- /dev/null +++ b/specs/telegram-gifts-indexer/data-model.md @@ -0,0 +1,37 @@ +# Data Model: Telegram Bot API Gift Indexing + +## 1. Core Entities + +### GiftUnique +Represents a specific, unique gift owned by a user (e.g., "SantaHat-1042"). + +**Fields**: +- `slug` (String, PK): The unique string identifier. Maps to Telegram `UniqueGift.name`. +- `collection_id` (String/Int, FK): References the parent collection. Maps to Telegram `UniqueGift.gift_id`. +- `number` (Int): The mint number of the gift. +- `model`, `backdrop`, `pattern` (String, Optional): Visual attributes. +- `telegram_owner_id` (Int, Optional): The Telegram user ID of the current owner. If null, the gift has no known owner on the platform. +- `blockchain_address`, `owner_address` (String, Optional): For blockchain-bridged gifts. + +**Relationships**: +- Belongs to one `GiftCollection` (via `collection_id`). +- Owned by one `User` (via `telegram_owner_id`). + +### Telegram API Models (via aiogram) + +#### `UserGifts` +Returned by `getUserGifts` endpoint. +- `total_count` (Int): Total number of gifts owned. +- `gifts` (List[`OwnedGiftUnique`]): The paginated list of gifts. + +#### `OwnedGiftUnique` +- `gift` (`UniqueGift`): The underlying gift definition. + +#### `UniqueGift` +- `gift_id` (String): Maps to `collection_id`. +- `name` (String): Maps to `slug`. + +## 2. Validation & State Transitions + +- **Gift Transfer/Loss**: If a `GiftUnique` in our database has `telegram_owner_id = X`, but user X's Telegram `getUserGifts` API response does NOT contain this gift's `name` (slug), the gift's `telegram_owner_id` must transition to `NULL`. +- **Gift Gain**: If `getUserGifts` contains a gift `name` not currently associated with user X, it must be upserted with `telegram_owner_id = X`. If it was previously owned by Y, Y loses it. diff --git a/specs/telegram-gifts-indexer/old-plan.md b/specs/telegram-gifts-indexer/old-plan.md new file mode 100644 index 00000000..e1a1cf0a --- /dev/null +++ b/specs/telegram-gifts-indexer/old-plan.md @@ -0,0 +1,53 @@ +# Plan: Telegram Bot API Gift Indexing + +## 1. Architecture Overview +The solution relies on Python (FastAPI + Celery) and PostgreSQL. It uses `aiogram` to communicate with the Telegram Bot API. +Tasks are segregated into prioritized Celery queues to guarantee responsive user-initiated indexing while allowing background syncs to run passively. + +## 2. Data Model Changes +- Align `GiftUniqueService` with the database schema: + - Methods (`get`, `find`, `create`, `update`, `update_ownership`) must use `slug` (string) as the primary identifier instead of `id` (integer). + - Remove deprecated filter arguments (`number_ge`, `number_le`). +- The Bot API payload mapping: + - `BotAPI.UniqueGift.name` -> `GiftUnique.slug` + - `BotAPI.UniqueGift.gift_id` -> `GiftUnique.collection_id` + +## 3. Queue & Worker Configuration +- Define queues in constants: + - `gift-user-queue` (Priority 1) + - `gift-fetch-queue` (Priority 2) + - `gift-background-queue` (Priority 3) +- Update Celery worker configuration in `docker-compose.yml` to listen to these three queues in order. +- Set `worker_prefetch_multiplier = 1` and `queue_order_strategy = "priority"` to enforce strict priority. + +## 4. API Endpoints +- **Endpoint**: `POST /api/users/me/gifts/refresh` +- **Dependency**: `validate_access_token` +- **Action**: `UserAction.refresh_gifts` +- **Rate Limit**: Uses Redis `SET nx ex=300`. Returns `{"status": "rate_limited"}` if locked. +- **Success**: Dispatches to `gift-user-queue` and returns `{"status": "pending", "task_id": "uuid"}`. + +## 5. Indexing Logic & Sync Flows +### Bot API Client +- Create a `BotApiGiftIndexer` leveraging `aiogram.Bot.get_user_gifts`. +- Implement an `AsyncGenerator` to yield pages of `OwnedGiftUnique` to prevent out-of-memory errors for users with thousands of gifts. + +### Flow B (User Initiated) +- `IndexerUserGiftAction.sync_user_gifts(user_id)`: + - Consumes the async generator. + - Upserts `GiftUnique` records using the `GiftUniqueService`. + - Links `telegram_owner_id` to the current user. + +### Flow C (Background Sync) +- **Master Task**: `refresh-all-user-gifts` runs hourly via Celery Beat. + - Fetches all user IDs. + - Batches them into chunks of 25. + - Dispatches `refresh-users-gifts-batch` tasks to `gift-background-queue`. +- **Loss Detection**: + - During sync, fetch all gifts currently owned by the user in the database. + - Compare against the active stream from the Bot API. + - If a database gift is not in the active stream, set `telegram_owner_id = None`. + - Push the user ID to the `UPDATED_GIFT_USER_IDS` Redis set for the community manager to verify chat eligibility. + +## 6. Deprecations +- Remove old Telethon-based tasks and indexers (`IndexerGiftCollectionAction`, `indexer_gifts/indexers/collection.py`, etc.). diff --git a/specs/telegram-gifts-indexer/plan.md b/specs/telegram-gifts-indexer/plan.md new file mode 100644 index 00000000..79759b8e --- /dev/null +++ b/specs/telegram-gifts-indexer/plan.md @@ -0,0 +1,72 @@ +# Implementation Plan: Telegram Bot API Gift Indexing + +**Branch**: `telegram-gifts-indexer` | **Date**: 2026-09-24 | **Spec**: [spec.md](spec.md) + +## Summary + +Track and index Telegram unique gifts via the official Telegram Bot API using three prioritized Celery queues. Replace the legacy Telethon implementation to properly index new user gifts on demand, and periodically background-sync all users to detect lost/transferred gifts to manage chat eligibility. + +## Technical Context + +**Language/Version**: Python 3.12+ + +**Primary Dependencies**: FastAPI, Celery, aiogram (3.24.0), SQLAlchemy + +**Storage**: PostgreSQL, Redis + +**Testing**: pytest + +**Target Platform**: Linux Server (Dockerized) + +**Project Type**: backend web service / background workers + +**Performance Goals**: Support users with thousands of gifts without OOM; fast background sync (<1 min per task chunk). + +**Constraints**: API rate limits from Telegram; Celery queue priority. + +**Scale/Scope**: ~10s of thousands of users. + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +No violations. + +## Project Structure + +### Documentation (this feature) + +```text +specs/telegram-gifts-indexer/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ # Phase 1 output +└── tasks.md # Phase 2 output (future) +``` + +### Source Code + +```text +backend/ +├── src/ +│ ├── core/ +│ │ ├── actions/ +│ │ ├── models/ +│ │ └── services/ +│ ├── api/ +│ │ └── routes/ +│ ├── indexer_gifts/ +│ │ ├── actions/ +│ │ ├── indexers/ +│ │ └── celery_app.py +│ └── scheduler/ +└── pyproject.toml +``` + +**Structure Decision**: Integrated directly into the existing `backend` monolithic FastAPI/Celery architecture. No new sub-projects needed. + +## Complexity Tracking + +N/A diff --git a/specs/telegram-gifts-indexer/quickstart.md b/specs/telegram-gifts-indexer/quickstart.md new file mode 100644 index 00000000..39649a48 --- /dev/null +++ b/specs/telegram-gifts-indexer/quickstart.md @@ -0,0 +1,42 @@ +# Quickstart & Validation Guide + +This guide describes how to validate the Telegram Bot API Gift Indexing functionality end-to-end. + +## Prerequisites +- A running local environment (`make run` or equivalent Docker Compose stack). +- An authenticated test user with a connected Telegram account that owns at least one unique gift. +- A mocked or valid Telegram Bot API token configured in the environment. + +## Scenario 1: Manual Refresh (Flow B) + +1. **Trigger Refresh**: + ```bash + curl -X POST http://localhost:8000/api/users/me/gifts/refresh \ + -H "Authorization: Bearer " + ``` + **Expected Outcome**: Returns HTTP 200 with `{"status": "pending", "task_id": "..."}`. + +2. **Verify Rate Limiting**: + Run the identical command immediately after. + **Expected Outcome**: Returns HTTP 200 with `{"status": "rate_limited"}`. + +3. **Verify Database Upsert**: + Check the `gift_unique` database table for the authenticated user. + **Expected Outcome**: The user's gifts appear with their `telegram_owner_id` properly set. + +## Scenario 2: Background Sync (Flow C) + +1. **Trigger Master Task manually**: + You can trigger the Celery task directly via a python shell or wait for the hourly scheduler. + ```bash + make test # Or use a script to invoke `refresh-all-user-gifts.delay()` + ``` + +2. **Simulate Gift Loss**: + - Manually assign a gift in the DB to the test user (`telegram_owner_id = `). + - Ensure the mocked Telegram API does *not* return this gift. + - Run the background sync task again. + +3. **Verify Revocation**: + - Check the database: The gift's `telegram_owner_id` should now be `NULL`. + - Check Redis: The set `UPDATED_GIFT_USER_IDS` should contain the user's ID. diff --git a/specs/telegram-gifts-indexer/research.md b/specs/telegram-gifts-indexer/research.md new file mode 100644 index 00000000..4d309d54 --- /dev/null +++ b/specs/telegram-gifts-indexer/research.md @@ -0,0 +1,15 @@ +# Research: Telegram Bot API Gift Indexing + +## Telegram Bot API Integration +- **Decision**: Use `aiogram` v3.24.0. +- **Rationale**: The official `aiogram` framework has built-in, type-safe support for all Telegram Bot API methods, including the new `getUserGifts` method and the `UniqueGift` type. This simplifies the payload parsing significantly compared to manual HTTP requests or raw `aiohttp`. +- **Alternatives considered**: Raw HTTP requests via `httpx` (rejected due to missing typing and boilerplate required) and legacy Telethon (rejected because Bot API handles this cleanly without full MTProto sessions). + +## Data Model Mismatch Resolution +- **Decision**: Align internal CRUD operations in `GiftUniqueService` to use `slug` instead of integer `id`. +- **Rationale**: The Bot API returns `name` (a string slug) as the unique identifier for a `UniqueGift`, whereas the older implementation attempted to use an integer `id`. Aligning the service to the database's actual primary key (`slug`) bridges the gap seamlessly. + +## Celery Task Prioritization +- **Decision**: Use `queue_order_strategy="priority"` with `worker_prefetch_multiplier=1` in Kombu/Celery. +- **Rationale**: We need three strict priority levels (Flow B > Flow A > Flow C). Without `prefetch_multiplier=1`, workers might reserve background tasks and block the priority queue. This native Kombu feature prevents starvation. +- **Alternatives considered**: Separate worker containers (rejected to save compute resources; 1 container can multiplex if configured correctly). diff --git a/specs/telegram-gifts-indexer/spec.md b/specs/telegram-gifts-indexer/spec.md new file mode 100644 index 00000000..9982a16a --- /dev/null +++ b/specs/telegram-gifts-indexer/spec.md @@ -0,0 +1,33 @@ +# Spec: Telegram Bot API Gift Indexing + +## 1. Description +A feature to track and index Telegram unique gifts owned by users through the official Telegram Bot API. This system will enable the platform to verify if a user possesses specific unique gifts required for membership in token-gated communities. + +## 2. Motivation +Currently, the application lacks a robust method for fetching unique gifts. Because unique gifts act as access passes for restricted communities, we need an automated, reliable system to ensure that: +1. New users get their gifts indexed immediately upon joining. +2. The system periodically checks all users to detect if they transferred or lost a required gift. +3. Users can manually request a refresh if they just acquired a gift. + +## 3. User Stories +- **As a new user**, when I join the application, my Telegram unique gifts are instantly indexed in the background without me having to do anything, so my chat eligibility is immediately evaluated. +- **As a user**, I can click a "Refresh Gifts" button on the UI, which will sync my latest gifts. To prevent spam, I can only do this once every 5 minutes. +- **As a community manager**, I want the system to continuously monitor chat members' gifts in the background, so that if a user transfers their required gift to someone else, they are automatically kicked from my gated chat. + +## 4. Requirements +### Functional +- The system must communicate with the Telegram Bot API (`getUserGifts`). +- The system must map Telegram's `UniqueGift` correctly to the internal database (associating Bot API `name` to our `slug`, and `gift_id` to `collection_id`). +- There must be a prioritized flow: User-initiated actions (Flow B) must take precedence over background syncs (Flow C). +- Background sync (Flow C) must cover all registered users on a configurable interval (default 1 hour). +- The system must detect when a user no longer owns a gift they previously held, revoke their ownership in the database, and trigger an eligibility recheck. + +### Non-Functional +- **Memory Safety**: The API client must process gifts page-by-page. It cannot load thousands of gifts into memory simultaneously. +- **Worker Availability**: Background tasks must be chunked (e.g., 25 users per task) so that Celery workers are not blocked for long periods. +- **Rate Limiting**: Manual refresh requests must be rate-limited at the API level (5 minutes). + +## 5. Out of Scope +- Telegram Stars tracking. +- Non-unique (regular) gift tracking. +- UI/Frontend implementation (this spec covers the backend functionality only). diff --git a/specs/telegram-gifts-indexer/tasks.md b/specs/telegram-gifts-indexer/tasks.md new file mode 100644 index 00000000..be071e0a --- /dev/null +++ b/specs/telegram-gifts-indexer/tasks.md @@ -0,0 +1,93 @@ +# Implementation Tasks: Telegram Bot API Gift Indexing + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Project configuration and basic structure. + +- [x] T001 Update `backend/indexer_gifts/pyproject.toml` or `requirements.txt` to include `aiogram==3.24.0`. +- [x] T002 Update `backend/core/src/core/constants.py` with new queue constants (`CELERY_GIFT_USER_QUEUE_NAME`, `CELERY_GIFT_FETCH_QUEUE_NAME`, `CELERY_GIFT_BACKGROUND_QUEUE_NAME`). +- [x] T003 Update `docker-compose.yml` to set Celery worker arguments for the indexer (`-Q gift-user-queue,gift-fetch-queue,gift-background-queue` and `prefetch_multiplier=1`). +- [x] T004 Delete unused legacy Telethon indexer code (`backend/indexer_gifts/actions/collection.py`, `backend/indexer_gifts/indexers/collection.py`). + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +- [x] T005 Update `GiftUniqueService` in `backend/core/src/core/services/gift/item.py` to use `slug` instead of `id` and remove legacy filters. (Note: partially done, but verify full alignment). +- [x] T006 Create `BotApiGiftIndexer` in `backend/indexer_gifts/indexers/bot_api.py` with an `AsyncGenerator` to yield pages of gifts efficiently. +- [x] T007 Implement core gift sync logic in `IndexerUserGiftAction.sync_user_gifts` in `backend/indexer_gifts/actions/user.py`, handling upserts and detection of lost gifts. + +**Checkpoint**: Foundation ready - user story implementation can now begin in parallel. + +--- + +## Phase 3: User Story 1 - Background Indexing for New Users (Priority: P1) 🎯 MVP + +**Goal**: As a new user, when I join the application, my Telegram unique gifts are instantly indexed in the background. + +**Independent Test**: Trigger a new user creation event or call `_initial_user_indexing` directly and observe the background task executing. + +### Implementation for User Story 1 + +- [x] T008 [US1] Create the `index-user-gifts` Celery task in `backend/indexer_gifts/tasks.py` that delegates to `IndexerUserGiftAction`. +- [x] T009 [US1] Add a call to trigger this indexing within `UserAction._initial_user_indexing` in `backend/core/src/core/actions/user.py`. + +**Checkpoint**: At this point, User Story 1 should be fully functional and testable independently. + +--- + +## Phase 4: User Story 2 - Manual Refresh with Rate Limiting (Priority: P2) + +**Goal**: As a user, I can click a "Refresh Gifts" button on the UI, which will sync my latest gifts. To prevent spam, I can only do this once every 5 minutes. + +**Independent Test**: Use the quickstart guide's curl command to verify rate-limiting and success responses. + +### Implementation for User Story 2 + +- [x] T010 [US2] Implement the `refresh_gifts` rate-limiting logic (using Redis) in `UserAction` inside `backend/core/src/core/actions/user.py`. +- [x] T011 [US2] Create the `POST /api/users/me/gifts/refresh` API endpoint in `backend/api/routes/user.py` conforming to the contract. + +**Checkpoint**: At this point, User Stories 1 AND 2 should both work independently. + +--- + +## Phase 5: User Story 3 - Background Sync (Priority: P3) + +**Goal**: As a community manager, I want the system to continuously monitor chat members' gifts in the background, so that if a user transfers their required gift to someone else, they are automatically kicked. + +**Independent Test**: Manually invoke the master task and verify users are chunked and background synced without blocking priority queues. + +### Implementation for User Story 3 + +- [x] T012 [US3] Create the master dispatcher task `refresh-all-user-gifts` and the worker batch task `refresh-users-gifts-batch` in `backend/indexer_gifts/tasks.py`. +- [x] T013 [US3] Add the hourly schedule for `refresh-all-user-gifts` in `backend/scheduler/celery_app.py`. + +**Checkpoint**: All user stories should now be independently functional. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Improvements that affect multiple user stories. + +- [x] T014 Run validation scenarios detailed in `specs/telegram-gifts-indexer/quickstart.md`. +- [x] T015 Verify Celery worker logs show correct prioritization of tasks. + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately. +- **Foundational (Phase 2)**: Depends on Setup completion - BLOCKS all user stories. +- **User Stories (Phase 3+)**: All depend on Foundational phase completion. + +### Parallel Opportunities + +- All Setup tasks (T001 - T004) can be done in parallel. +- US2 API endpoints (T011) can be defined in parallel to US1 task creation (T008).