From f94d095db326abb4c4b57679ac82b7625adb93fc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 10:15:47 +0000 Subject: [PATCH 1/6] Add request size limits and a wall-clock match timeout /api/match and /api/generate accept arbitrary user-supplied patterns and log text, with no bound on either size or how long matching can take. Two independent mitigations: - DebugRequest now caps pattern (5,000), custom_patterns (20,000) and log_text (200,000) length via pydantic Field(max_length=...). - Both engine calls now run off the event loop through a small ThreadPoolExecutor with a wall-clock budget (MATCH_TIMEOUT_SECONDS, default 3s, env-overridable), via a new _run_with_timeout() helper. This bounds *response time*, not CPU usage - see the comment above MATCH_TIMEOUT_SECONDS for the full reasoning, including why pygrok's use of the third-party 'regex' package (one of its own dependencies) already mitigates some classic catastrophic-backtracking patterns before this timeout would ever need to fire. Fixes a real bug this surfaces: pydantic's default validation-error body is a list of error objects, which the frontend's `this.errorMessage = data.detail || ...` can't render sensibly (it would show "[object Object]"). Added a RequestValidationError handler that flattens it to the same single-string {"detail": "..."} shape HTTPException responses already use, so a validation failure (missing field, oversized input, etc.) displays exactly like a 400 does. --- app/main.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/app/main.py b/app/main.py index 3c0aa16..8b36dd2 100644 --- a/app/main.py +++ b/app/main.py @@ -1,9 +1,15 @@ +import asyncio +import functools import logging +import os +from concurrent.futures import ThreadPoolExecutor from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -from pydantic import BaseModel +from pydantic import BaseModel, Field from app.config import settings from app.grok_engine import GrokDebuggerEngine @@ -22,6 +28,97 @@ templates = Jinja2Templates(directory="app/templates") app.mount("/static", StaticFiles(directory="app/static"), name="static") +# Request size limits. A Grok/regex pattern or sample log pasted by mistake +# (or crafted deliberately) can be arbitrarily large; these bound the request +# body pydantic will accept before it ever reaches the matching engine. +MAX_PATTERN_LENGTH = 5_000 +MAX_CUSTOM_PATTERNS_LENGTH = 20_000 +MAX_LOG_TEXT_LENGTH = 200_000 + +# Grok/regex matching is CPU-bound and runs off the event loop with a hard +# wall-clock budget: a pathological pattern (catastrophic backtracking) could +# otherwise hang a request - and, since uvicorn's default worker is +# single-threaded for the event loop, every other in-flight request - forever. +# +# Note pygrok prefers the third-party `regex` package over stdlib `re` when +# it's installed (it's one of pygrok's own dependencies, so in practice it +# normally is - see `pip show regex`), and `regex` already handles some +# classically pathological patterns (e.g. `(a+)+` against a run of matching +# characters) far better than stdlib `re`. This timeout is still worth +# keeping regardless: it's a single wall-clock budget around the *entire* +# engine call - not just one regex operation - so it also covers +# find_partial_match's loop of progressively-longer sub-pattern compiles and +# pregenerate_pattern's diff-based alignment, neither of which is a single +# regex op a timeout=... kwarg could bound on its own. +# +# What it does NOT do is bound CPU usage: Python cannot forcibly interrupt a +# C-level regex match mid-flight, so if the underlying call ever is slow +# enough to hit this timeout, that worker thread keeps running in the +# background even after asyncio.wait_for gives up on it and returns an error +# to the caller. Combined with the size limits above, this keeps the API +# responsive for trusted/internal use; it is not a complete DoS defense for +# an untrusted multi-tenant deployment. +_match_executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="grok-match") +MATCH_TIMEOUT_SECONDS = float(os.getenv("MATCH_TIMEOUT_SECONDS", "3")) + + +async def _run_with_timeout(func, *args, **kwargs): + """ + Run a blocking engine call off the event loop with a wall-clock timeout. + + Args: + func: The blocking callable to run (e.g. engine.execute_match). + *args: Positional arguments to pass to func. + **kwargs: Keyword arguments to pass to func. + + Returns: + The return value of func. + + Raises: + ValueError: If func does not complete within MATCH_TIMEOUT_SECONDS. + """ + loop = asyncio.get_running_loop() + call = functools.partial(func, *args, **kwargs) + try: + return await asyncio.wait_for( + loop.run_in_executor(_match_executor, call), + timeout=MATCH_TIMEOUT_SECONDS + ) + except asyncio.TimeoutError as e: + raise ValueError( + "Pattern evaluation timed out - the pattern or input is likely too " + "complex (possible catastrophic backtracking). Try simplifying the " + "pattern or reducing the sample size." + ) from e + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + _request: Request, exc: RequestValidationError +) -> JSONResponse: + """ + Flatten pydantic's default multi-error body into the single-string + `{"detail": "..."}` shape the frontend already expects from + HTTPException responses (see app.js's triggerMatch/autoGenerate), + so a request-size violation displays the same way a 400 does instead + of rendering "[object Object]". + + Args: + _request: The FastAPI request that failed validation (unused; + required by FastAPI's exception handler signature). + exc: The validation error raised by pydantic. + + Returns: + A 422 JSON response with a single human-readable `detail` string. + """ + first_error = exc.errors()[0] if exc.errors() else {} + field = ".".join(str(p) for p in first_error.get("loc", []) if p != "body") + message = first_error.get("msg", "Invalid request.") + detail = f"{field}: {message}" if field else message + logger.warning(f"Request validation error: {detail}") + return JSONResponse(status_code=422, content={"detail": detail}) + + class DebugRequest(BaseModel): """ Request model for Grok debugging API endpoints. @@ -35,9 +132,9 @@ class DebugRequest(BaseModel): strict_mode: If True, require a full line match (^...$). Otherwise, allow substring matches. """ - pattern: str - custom_patterns: str | None = "" - log_text: str + pattern: str = Field(max_length=MAX_PATTERN_LENGTH) + custom_patterns: str | None = Field(default="", max_length=MAX_CUSTOM_PATTERNS_LENGTH) + log_text: str = Field(max_length=MAX_LOG_TEXT_LENGTH) naming_format: str | None = "dot" strict_mode: bool | None = False @@ -84,7 +181,8 @@ async def match_grok(data: DebugRequest): try: # Explicitly handle empty custom_patterns custom_patterns = data.custom_patterns if data.custom_patterns else "" - matches = engine.execute_match( + matches = await _run_with_timeout( + engine.execute_match, data.pattern, custom_patterns, data.log_text, @@ -92,7 +190,7 @@ async def match_grok(data: DebugRequest): ) return {"success": True, "results": matches} except ValueError as e: - logger.warning(f"Validation error in match_grok: {e}") + logger.warning(f"match_grok request rejected: {e}") raise HTTPException(status_code=400, detail=str(e)) from e except Exception as e: logger.exception("Unexpected error in match_grok") @@ -116,7 +214,8 @@ async def generate_pattern(data: DebugRequest): - error: Error message (if failed). """ try: - guessed_pattern = engine.pregenerate_pattern( + guessed_pattern = await _run_with_timeout( + engine.pregenerate_pattern, data.log_text, data.naming_format or "dot" ) From bc0baf9e7afdc4a8b2ce3e7735d9d4da50ab2731 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 10:15:53 +0000 Subject: [PATCH 2/6] Add a pytest test suite The project had no tests at all. Adds: - tests/test_grok_engine.py: unit tests for GrokDebuggerEngine - matching (grok fields, named regex groups, strict mode, custom pattern definitions), field-name validation, type detection, dict flattening/unflattening, and pattern auto-generation. Includes explicit regression tests for the find_partial_match tokenizer bug fixed earlier (both the simple named-group case and the harder nested-parens-inside-a-group case). - tests/test_api.py: FastAPI TestClient tests for all four endpoints, including the new request size limits, the flattened validation- error shape, and the match timeout (mocked rather than relying on a genuinely catastrophic pattern, since pygrok's preference for the regex package over stdlib re makes wall-clock ReDoS timing environment-dependent - see the test's docstring). - requirements-dev.txt and a [project.optional-dependencies] dev group in pyproject.toml (ruff, bandit, pytest, httpx). - [tool.pytest.ini_options] in pyproject.toml so resolves the app package correctly regardless of invocation style. --- pyproject.toml | 13 +++ requirements-dev.txt | 5 + tests/test_api.py | 151 +++++++++++++++++++++++++++++ tests/test_grok_engine.py | 197 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 366 insertions(+) create mode 100644 requirements-dev.txt create mode 100644 tests/test_api.py create mode 100644 tests/test_grok_engine.py diff --git a/pyproject.toml b/pyproject.toml index 1a12d6b..34756e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,14 @@ dependencies = [ "pydantic>=2.0.0" ] +[project.optional-dependencies] +dev = [ + "ruff>=0.6.0", + "bandit>=1.7.0", + "pytest>=8.0.0", + "httpx>=0.27.0" +] + [tool.ruff] line-length = 100 @@ -21,3 +29,8 @@ line-length = 100 # N: naming. ARG: unused arguments. BLE: no bare `except Exception` without # a justifying comment/noqa (see grok_engine.py's find_partial_match). select = ["E", "F", "W", "B", "C4", "SIM", "UP", "RUF", "PTH", "N", "ARG", "BLE"] + +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["tests"] + diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7a17f8e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +-r requirements.txt +ruff>=0.6.0 +bandit>=1.7.0 +pytest>=8.0.0 +httpx>=0.27.0 diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..3b7cb91 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,151 @@ +"""Tests for the FastAPI endpoints in app.main.""" +import time + +import app.main as main_module +from fastapi.testclient import TestClient + +client = TestClient(main_module.app) + + +def test_health_check_reports_healthy(): + res = client.get("/api/health") + assert res.status_code == 200 + assert res.json() == {"status": "healthy"} + + +def test_get_config_returns_version_and_features(): + res = client.get("/api/config") + assert res.status_code == 200 + body = res.json() + assert "version" in body + assert "features" in body + + +def test_index_page_renders(): + res = client.get("/") + assert res.status_code == 200 + assert "text/html" in res.headers["content-type"] + + +def test_match_returns_extracted_fields_for_a_valid_pattern(): + res = client.post( + "/api/match", + json={ + "pattern": "%{IP:client_ip} %{WORD:method} %{NUMBER:status}", + "custom_patterns": "", + "log_text": "192.168.1.1 GET 200", + "naming_format": "dot", + "strict_mode": False, + }, + ) + assert res.status_code == 200 + body = res.json() + assert body["success"] is True + assert body["results"][0]["matches"] == { + "client_ip": "192.168.1.1", + "method": "GET", + "status": "200", + } + + +def test_match_reports_a_partial_match_for_a_mixed_grok_and_regex_pattern(): + """Regression test: this used to return an empty partial_match due to a + broken tokenizer in find_partial_match (see app/grok_engine.py).""" + res = client.post( + "/api/match", + json={ + "pattern": r"(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (?P[0-9]{3})", + "custom_patterns": "", + "log_text": "192.168.1.1 wrong-status", + "naming_format": "dot", + "strict_mode": False, + }, + ) + assert res.status_code == 200 + partial = res.json()["results"][0]["partial_match"] + assert partial["matched_fields"] == {"ip": "192.168.1.1"} + + +def test_match_with_invalid_pattern_returns_400_with_string_detail(): + res = client.post( + "/api/match", + json={ + "pattern": "%{NOT_A_REAL_GROK_TYPE:x}", + "custom_patterns": "", + "log_text": "some text", + }, + ) + assert res.status_code == 400 + assert isinstance(res.json()["detail"], str) + + +def test_match_missing_required_field_returns_422_with_flattened_string_detail(): + """Regression test: pydantic's default validation-error body is a list of + objects, which the frontend can't render as-is (see the + RequestValidationError handler in app/main.py).""" + res = client.post("/api/match", json={"custom_patterns": ""}) + assert res.status_code == 422 + assert isinstance(res.json()["detail"], str) + + +def test_match_pattern_over_the_length_limit_is_rejected(): + res = client.post( + "/api/match", + json={ + "pattern": "a" * (main_module.MAX_PATTERN_LENGTH + 1), + "custom_patterns": "", + "log_text": "some text", + }, + ) + assert res.status_code == 422 + assert isinstance(res.json()["detail"], str) + + +def test_generate_pattern_produces_a_pattern_that_matches_its_own_sample(): + log_line = "2024-01-01T10:00:00 192.168.1.1 GET 200" + gen_res = client.post( + "/api/generate", + json={"pattern": "", "custom_patterns": "", "log_text": log_line}, + ) + assert gen_res.status_code == 200 + gen_body = gen_res.json() + assert gen_body["success"] is True + + match_res = client.post( + "/api/match", + json={ + "pattern": gen_body["generated_pattern"], + "custom_patterns": "", + "log_text": log_line, + }, + ) + assert match_res.json()["results"][0]["matched"] is True + + +def test_match_times_out_when_the_engine_call_takes_too_long(monkeypatch): + """The timeout wrapper must fail fast with a clear message rather than + hang the request indefinitely. + + This mocks a slow engine call instead of relying on a genuinely + catastrophic-backtracking pattern: pygrok prefers the third-party + `regex` package over stdlib `re` when it's installed (it's one of + pygrok's own dependencies, so it normally is), and `regex` handles some + classically pathological patterns - like (a+)+ against a run of "a"s - + far better than stdlib re does. Testing against a specific pattern's + wall-clock time would make this test flaky across environments; testing + the timeout mechanism itself does not. + """ + monkeypatch.setattr(main_module, "MATCH_TIMEOUT_SECONDS", 0.1) + + def slow_execute_match(*_args, **_kwargs): + time.sleep(1) + return [] + + monkeypatch.setattr(main_module.engine, "execute_match", slow_execute_match) + + res = client.post( + "/api/match", + json={"pattern": "%{IP:ip}", "custom_patterns": "", "log_text": "192.168.1.1"}, + ) + assert res.status_code == 400 + assert "timed out" in res.json()["detail"].lower() diff --git a/tests/test_grok_engine.py b/tests/test_grok_engine.py new file mode 100644 index 0000000..db6898a --- /dev/null +++ b/tests/test_grok_engine.py @@ -0,0 +1,197 @@ +"""Unit tests for app.grok_engine.GrokDebuggerEngine.""" +import pytest + +from app.grok_engine import GrokDebuggerEngine + + +@pytest.fixture +def engine() -> GrokDebuggerEngine: + return GrokDebuggerEngine() + + +class TestExecuteMatch: + def test_matches_a_simple_grok_pattern(self, engine): + results = engine.execute_match( + "%{IP:client_ip} %{WORD:method} %{NUMBER:status}", + "", + "192.168.1.1 GET 200", + ) + assert len(results) == 1 + assert results[0]["matched"] is True + assert results[0]["matches"] == { + "client_ip": "192.168.1.1", + "method": "GET", + "status": "200", + } + + def test_dot_notation_fields_are_unflattened_into_nested_json(self, engine): + results = engine.execute_match( + "%{IP:client.ip} %{NUMBER:client.port}", + "", + "10.0.0.1 8080", + ) + assert results[0]["json_data"] == {"client": {"ip": "10.0.0.1", "port": "8080"}} + + def test_bracket_notation_fields_are_unflattened_into_nested_json(self, engine): + results = engine.execute_match( + "%{IP:[client][ip]} %{NUMBER:[client][port]}", + "", + "10.0.0.1 8080", + ) + assert results[0]["json_data"] == {"client": {"ip": "10.0.0.1", "port": "8080"}} + + def test_named_regex_groups_work_alongside_grok_fields(self, engine): + results = engine.execute_match( + r"%{IP:client_ip} (?P[0-9]{3})", + "", + "192.168.1.1 200", + ) + assert results[0]["matched"] is True + assert results[0]["matches"] == {"client_ip": "192.168.1.1", "status": "200"} + + def test_unmatched_line_reports_no_matches(self, engine): + results = engine.execute_match("%{IP:client_ip}", "", "not-an-ip at all") + assert results[0]["matched"] is False + assert results[0]["matches"] == {} + + def test_strict_mode_requires_a_full_line_match(self, engine): + loose = engine.execute_match("%{IP:ip}", "", "192.168.1.1 extra text") + strict = engine.execute_match( + "%{IP:ip}", "", "192.168.1.1 extra text", strict_mode=True + ) + assert loose[0]["matched"] is True + assert strict[0]["matched"] is False + + def test_blank_lines_are_skipped(self, engine): + results = engine.execute_match("%{IP:ip}", "", "192.168.1.1\n\n10.0.0.1") + assert [r["line_number"] for r in results] == [1, 3] + + def test_empty_pattern_or_text_returns_no_results(self, engine): + assert engine.execute_match("", "", "some text") == [] + assert engine.execute_match("%{IP:ip}", "", "") == [] + + def test_invalid_pattern_raises_value_error(self, engine): + with pytest.raises(ValueError, match="Pattern Compilation Error"): + engine.execute_match("%{NOT_A_REAL_GROK_TYPE:x}", "", "some text") + + def test_custom_pattern_definitions_are_honored(self, engine): + results = engine.execute_match( + "%{MY_ID:id}", + "MY_ID [a-z]+-[0-9]+", + "abc-123", + ) + assert results[0]["matched"] is True + assert results[0]["matches"] == {"id": "abc-123"} + + +class TestFindPartialMatch: + """Regression tests for the tokenizer bug in find_partial_match. + + The tokenizer used to split (?P...) into two invalid fragments + ('(?P' and '...)'), so any progressively-built prefix containing + one of those fragments failed to compile and the diagnostic silently + reported no partial match at all. + """ + + def test_reports_the_longest_matching_prefix_for_a_named_regex_group(self, engine): + result = engine.find_partial_match( + r"(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) (?P[0-9]{3})", + {}, + "192.168.1.1 wrong-status", + ) + assert result["matched_fields"] == {"ip": "192.168.1.1"} + assert result["unmatched_remainder"] == "wrong-status" + + def test_handles_nested_parens_inside_a_named_group_body(self, engine): + result = engine.find_partial_match( + r"(?P(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}) (?P[0-9]{3})", + {}, + "00:1A:2B:3C:4D:5E nope", + ) + assert result["matched_fields"] == {"mac": "00:1A:2B:3C:4D:5E"} + assert result["unmatched_remainder"] == "nope" + + def test_reports_the_longest_matching_prefix_for_a_grok_field(self, engine): + result = engine.find_partial_match( + "%{IP:ip} %{WORD:method} %{NUMBER:status}", + {}, + "192.168.1.1 GET not-a-number", + ) + assert result["matched_fields"] == {"ip": "192.168.1.1", "method": "GET"} + + def test_no_match_at_all_reports_empty_prefix(self, engine): + result = engine.find_partial_match("%{IP:ip}", {}, "definitely not an ip") + assert result["matched_prefix"] == "" + assert result["matched_fields"] == {} + assert result["unmatched_remainder"] == "definitely not an ip" + + +class TestFieldNameValidation: + def test_accepts_dot_notation(self, engine): + engine.validate_logstash_field_name("client.ip") # does not raise + + def test_accepts_bracket_notation(self, engine): + engine.validate_logstash_field_name("[client][ip]") # does not raise + + def test_rejects_mixed_or_malformed_notation(self, engine): + with pytest.raises(ValueError, match="Invalid Logstash field name"): + engine.validate_logstash_field_name("client.[ip]") + + +class TestDetectGrokType: + @pytest.mark.parametrize( + ("value", "expected_type"), + [ + ("192.168.1.1", "IP"), + ("2001:db8::1", "IP"), + ("/var/log/syslog", "PATH"), + ("2024-01-01T10:00:00", "TIMESTAMP_ISO8601"), + ("42", "INT"), + ("3.14", "NUMBER"), + ("hello", "NOTSPACE"), + ("10:23:01", "NOTSPACE"), # plain HH:MM:SS, not IPv6 + ], + ) + def test_detects_expected_type(self, engine, value, expected_type): + assert engine.detect_grok_type(value) == expected_type + + +class TestUnflattenDict: + def test_unflattens_dot_and_bracket_keys_together(self, engine): + flat = {"client.ip": "192.168.1.1", "[server][port]": "80"} + assert engine.unflatten_dict(flat) == { + "client": {"ip": "192.168.1.1"}, + "server": {"port": "80"}, + } + + +class TestParseCustomPatterns: + def test_parses_name_and_pattern_pairs_and_skips_comments(self, engine): + raw = "# a comment\nMY_ID [a-z]+-[0-9]+\n\nOTHER \\d+" + assert engine.parse_custom_patterns(raw) == { + "MY_ID": "[a-z]+-[0-9]+", + "OTHER": "\\d+", + } + + +class TestPregeneratePattern: + def test_empty_text_falls_back_to_greedydata(self, engine): + assert engine.pregenerate_pattern("") == "%{GREEDYDATA:message}" + + def test_generated_pattern_matches_the_sample_it_was_built_from(self, engine): + log_line = "2024-01-01T10:00:00 192.168.1.1 GET 200" + pattern = engine.pregenerate_pattern(log_line) + results = engine.execute_match(pattern, "", log_line) + assert results[0]["matched"] is True + + def test_single_sample_line_has_nothing_to_diff_against_and_stays_literal(self, engine): + # pregenerate_pattern detects *variable* content by diffing multiple + # sample lines; a lone "key=value" line has nothing to compare + # against, so it's correctly returned as a literal, not a field. + assert engine.pregenerate_pattern("user=alice") == "user=alice" + + def test_bracket_format_mode_produces_bracket_field_names(self, engine): + pattern = engine.pregenerate_pattern( + "user=alice\nuser=bob", format_mode="bracket" + ) + assert "[user]" in pattern From 231b6f7233df9a3940827a06be240cd71ee94f88 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 10:16:00 +0000 Subject: [PATCH 3/6] Add CI: run ruff, bandit and pytest on push and PR .github/workflows/ci.yml installs requirements-dev.txt and runs 'ruff check app/ tests/', 'bandit -r app/' and 'pytest -v' on every push to main and every pull request, across Python 3.10-3.12. --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ca521ba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install dependencies + run: pip install -r requirements-dev.txt + + - name: Lint (ruff) + run: ruff check app/ tests/ + + - name: Security scan (bandit) + run: bandit -r app/ + + - name: Test (pytest) + run: pytest -v From 2a8dfcf7fcd1baab0fb203b006736ec2371ba67e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 11:40:42 +0000 Subject: [PATCH 4/6] Update README: dev/test docs, request limits, production notes, license - Project Structure now lists .devcontainer/, .github/workflows/, tests/, pyproject.toml and requirements-dev.txt. - New Development section: how to install dev extras and run pytest, ruff and bandit, and a pointer to the CI workflow that runs them. - New Request Limits & Timeouts section documenting the size caps and match timeout added earlier, with a pointer to the fuller reasoning in app/main.py. - New Production Notes section flagging the two frontend CDN trade-offs (Tailwind's Play CDN, Alpine's floating version/missing SRI) with concrete steps to harden them - not applied automatically, since a wrong SRI hash would break the app outright. - New License section - the repo is GPLv3 (LICENSE) but the README never said so. --- README.md | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f831b0d..02f17d9 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ This project was built to modernize and combine capabilities of Grok debugging t ``` . +├── .devcontainer/ +│ └── devcontainer.json # VS Code dev container config +├── .github/ +│ └── workflows/ +│ └── ci.yml # Lint (ruff), security scan (bandit) & tests (pytest) on push/PR ├── app/ │ ├── main.py # FastAPI application & API routes │ ├── config.py # App settings & version/feature metadata @@ -38,7 +43,12 @@ This project was built to modernize and combine capabilities of Grok debugging t │ └── static/ │ ├── css/style.css # UI styling │ └── js/app.js # Alpine.js application logic -├── requirements.txt # Python dependencies +├── tests/ +│ ├── test_grok_engine.py # Unit tests for the matching/generation engine +│ └── test_api.py # API-level tests (FastAPI TestClient) +├── pyproject.toml # Project metadata, ruff config, pytest config +├── requirements.txt # Runtime dependencies +├── requirements-dev.txt # + ruff, bandit, pytest, httpx └── README.md ``` @@ -72,3 +82,72 @@ uvicorn app.main:app --reload --port 8000 ``` 4. Open [http://localhost:8000](http://localhost:8000) in your browser. + +--- + +## Development + +Install the dev extras (adds ruff, bandit, pytest, httpx on top of the runtime dependencies): + +```bash +pip install -r requirements-dev.txt +``` + +**Run the tests:** + +```bash +pytest +``` + +**Lint:** + +```bash +ruff check app/ tests/ +``` + +The lint ruleset is pinned in `pyproject.toml` under `[tool.ruff.lint]` so `ruff check` gives the same result locally and in CI. + +**Security scan:** + +```bash +bandit -r app/ +``` + +All three run automatically on every push and pull request via [`.github/workflows/ci.yml`](.github/workflows/ci.yml), across Python 3.10–3.12. + +--- + +## Request Limits & Timeouts + +`/api/match` and `/api/generate` accept arbitrary user-supplied patterns and sample text, so a few limits are enforced server-side (see `app/main.py`): + +| Limit | Default | Purpose | +|---|---|---| +| Pattern length | 5,000 chars | Bounds pathological input | +| Custom pattern definitions length | 20,000 chars | Bounds pathological input | +| Log text length | 200,000 chars | Bounds pathological input | +| Match/generate timeout | 3s (`MATCH_TIMEOUT_SECONDS` env var) | Fails fast instead of hanging a request | + +The timeout bounds *response time*, not CPU usage — Python can't forcibly interrupt a regex match mid-flight, so it's a practical safeguard for trusted/internal use rather than a complete denial-of-service defense. See the comment above `MATCH_TIMEOUT_SECONDS` in `app/main.py` for the full reasoning, including why pygrok's use of the third-party `regex` package (installed automatically as one of pygrok's own dependencies) already mitigates some classic catastrophic-backtracking patterns on its own. + +--- + +## Production Notes + +The frontend is intentionally dependency-light for easy self-hosting, with two trade-offs worth knowing about before deploying it beyond a trusted local/internal setting: + +- **Tailwind via the CDN script** (`cdn.tailwindcss.com`) is Tailwind's "Play CDN" — convenient for zero-build local use, but Tailwind's own docs note it isn't intended for production (it compiles styles in the browser on every load). For a production build, compile Tailwind via its CLI or a bundler and link the resulting stylesheet instead. +- **Alpine.js is loaded from a floating version range** (`alpinejs@3.x.x`). Consider pinning it to an exact version (e.g. `alpinejs@3.17.1`) and adding a [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) hash, e.g.: + + ```bash + curl -s https://cdn.jsdelivr.net/npm/alpinejs@3.17.1/dist/cdn.min.js \ + | openssl dgst -sha384 -binary | openssl base64 -A + ``` + + and add `integrity="sha384-" crossorigin="anonymous"` to the `