diff --git a/.gitignore b/.gitignore index 6c3f0379..f1534e69 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ -# Benchmark generated output -benchmarks/results/ -benchmarks/figures/ - ################# ## Eclipse ################# diff --git a/AGENTS.md b/AGENTS.md index 9ee4ce07..93af4158 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,7 +49,7 @@ CI runs ruff check, ruff format --check, typos, the unittest suite, and the spec ## Git flow -Hosted at hed-standard; `origin` is the VisLab fork and `upstream` is hed-standard. Local `main` is a clean mirror of `upstream/main` - never commit or merge to it locally. All work goes on a branch based on `upstream/main`, pushed to the fork, and merged via a PR to hed-standard. +Hosted at https://github.com/hed-standard/hed-python. Keep local `main` a clean mirror of the hed-standard `main` - never commit or merge to it locally. Do all work on a branch based on that `main` and get it into hed-standard through a pull request, typically pushed to your own fork first. Remote names (`origin`, `upstream`, ...) vary by checkout, so commands here never assume them; your own remote layout is a machine fact for `.status/local-environment.md`. ## Related repositories diff --git a/README.md b/README.md index 333b077d..35746a09 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ See [`examples/README.md`](examples/README.md) for more details. > from hed.models.schema_lookup import generate_schema_lookup > ``` -See the [search details documentation](https://www.hedtags.org/hed-python/search_details.html) for a full comparison of all three search implementations and performance benchmarks. +See the [search details documentation](https://www.hedtags.org/hed-python/search_details.html) for a full comparison of all three search implementations and performance benchmarks. The benchmark harness itself now lives in the [hed-benchmarks](https://github.com/hed-standard/hed-benchmarks) repository. ## Documentation diff --git a/benchmarks/data_generator.py b/benchmarks/data_generator.py deleted file mode 100644 index 975cd8cc..00000000 --- a/benchmarks/data_generator.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Generate synthetic and real HED strings/Series for benchmarking. - -Usage:: - - from data_generator import DataGenerator - gen = DataGenerator() # loads schema 8.4.0 - s = gen.make_string(n_tags=10, n_groups=2, depth=1) - series = gen.make_series(n_rows=1000, n_tags=10, n_groups=2, depth=1) - real = gen.load_real_data(tile_to=5000) -""" - -from __future__ import annotations - -import os -import random - -import pandas as pd - -from hed.models.df_util import convert_to_form -from hed.models.schema_lookup import generate_schema_lookup -from hed.models.tabular_input import TabularInput -from hed.schema import load_schema_version - - -class DataGenerator: - """Build synthetic and real HED data for benchmarking.""" - - def __init__(self, schema_version="8.4.0", seed=42): - self.schema = load_schema_version(schema_version) - self.lookup = generate_schema_lookup(self.schema) - self._rng = random.Random(seed) - - # Collect real tag short names from the schema for realistic generation - self._all_tags = [] - for name, entry in self.schema.tags.items(): - if name.endswith("/#"): - continue - short = getattr(entry, "short_tag_name", name.rsplit("/", 1)[-1]) - self._all_tags.append(short) - - # Separate leaf vs non-leaf for variety - self._tags = list(self._all_tags) - - # ------------------------------------------------------------------ - # Single string generation - # ------------------------------------------------------------------ - - def _pick_tags(self, n, repeats=0): - """Pick *n* unique tags, then append *repeats* duplicates of the first.""" - chosen = self._rng.sample(self._tags, min(n, len(self._tags))) - if repeats and chosen: - chosen.extend([chosen[0]] * repeats) - return chosen - - def make_string(self, n_tags=5, n_groups=0, depth=0, repeats=0, form="short"): - """Build a single synthetic HED string. - - Parameters: - n_tags: Total number of tag tokens (spread across top-level and groups). - n_groups: Number of parenthesised groups to create. - depth: Maximum nesting depth inside groups. - repeats: Number of duplicate copies of the first tag to append. - form: 'short' | 'long' — tag form. - - Returns: - str: A raw HED string. - """ - tags = self._pick_tags(n_tags, repeats=repeats) - if form == "long": - tags = self._to_long(tags) - - if n_groups == 0 or depth == 0: - return ", ".join(tags) - - # Distribute tags across top-level and groups - top_count = max(1, n_tags - n_groups * 2) - top_tags = tags[:top_count] - remaining = tags[top_count:] - - parts = list(top_tags) - for i in range(n_groups): - group_tags = remaining[i * 2 : i * 2 + 2] if i * 2 + 2 <= len(remaining) else remaining[i * 2 :] - if not group_tags: - group_tags = [self._rng.choice(self._tags)] - parts.append(self._wrap_group(group_tags, depth)) - - return ", ".join(parts) - - def _wrap_group(self, tags, depth): - """Recursively nest *tags* to the given *depth*.""" - inner = ", ".join(tags) - result = f"({inner})" - for _ in range(depth - 1): - extra = self._rng.choice(self._tags) - result = f"({extra}, {result})" - return result - - def make_deeply_nested_string(self, depth, tags_per_level=2): - """Build a string with deep nesting: (A, (B, (C, ...))). - - Parameters: - depth: Number of nesting levels. - tags_per_level: Tags at each level. - - Returns: - str: Deeply nested HED string. - """ - tags = self._pick_tags(depth * tags_per_level + 2) - # Build inside-out - inner = ", ".join(tags[:tags_per_level]) - for i in range(depth): - level_tags = tags[tags_per_level + i * tags_per_level : tags_per_level + (i + 1) * tags_per_level] - if not level_tags: - level_tags = [self._rng.choice(self._tags)] - inner = f"({', '.join(level_tags)}, ({inner}))" - return f"Event, Action, {inner}" - - def make_string_with_specific_tags(self, target_tags, n_extra=5, n_groups=2, depth=1, repeats=0): - """Build a string guaranteed to contain specific tags. - - Parameters: - target_tags: List of tag names to include. - n_extra: Number of random extra tags. - n_groups: Number of groups. - depth: Nesting depth. - repeats: How many times to repeat the first target tag. - - Returns: - str: HED string containing the target tags. - """ - extra = self._pick_tags(n_extra) - all_tags = list(target_tags) + extra + [target_tags[0]] * repeats - self._rng.shuffle(all_tags) - - if n_groups == 0 or depth == 0: - return ", ".join(all_tags) - - top_count = max(1, len(all_tags) - n_groups * 2) - top_tags = all_tags[:top_count] - remaining = all_tags[top_count:] - - parts = list(top_tags) - for i in range(n_groups): - group_tags = remaining[i * 2 : i * 2 + 2] if i * 2 + 2 <= len(remaining) else remaining[i * 2 :] - if not group_tags: - group_tags = [self._rng.choice(self._tags)] - parts.append(self._wrap_group(group_tags, depth)) - - return ", ".join(parts) - - def _to_long(self, short_tags): - """Convert short tag names to long form via the schema.""" - from hed.models.hed_tag import HedTag - - out = [] - for t in short_tags: - try: - out.append(HedTag(t, self.schema).long_tag) - except Exception: - out.append(t) - return out - - # ------------------------------------------------------------------ - # Series generation - # ------------------------------------------------------------------ - - def make_series(self, n_rows, *, n_tags=5, n_groups=0, depth=0, repeats=0, form="short", heterogeneous=False): - """Build a pd.Series of HED strings. - - Parameters: - n_rows: Number of rows. - n_tags, n_groups, depth, repeats, form: Passed to make_string. - heterogeneous: If True, randomise parameters per row. - """ - if heterogeneous: - rows = [] - for _ in range(n_rows): - nt = self._rng.choice([3, 5, 10, 15, 25]) - ng = self._rng.choice([0, 1, 2, 5]) - d = self._rng.choice([0, 1, 2]) - rows.append(self.make_string(n_tags=nt, n_groups=ng, depth=d, form=form)) - return pd.Series(rows) - else: - # Homogeneous: one template, tiled - template = self.make_string(n_tags=n_tags, n_groups=n_groups, depth=depth, repeats=repeats, form=form) - return pd.Series([template] * n_rows) - - # ------------------------------------------------------------------ - # Real data - # ------------------------------------------------------------------ - - def load_real_data(self, tile_to=None, form="short"): - """Load the FacePerception BIDS events and return a HED Series. - - Parameters: - tile_to: If set, tile the series up to this many rows. - form: 'short' | 'long'. - - Returns: - pd.Series of HED strings. - """ - bids_root = os.path.realpath( - os.path.join(os.path.dirname(__file__), "..", "tests", "data", "bids_tests", "eeg_ds003645s_hed") - ) - sidecar = os.path.join(bids_root, "task-FacePerception_events.json") - events = os.path.join(bids_root, "sub-002", "eeg", "sub-002_task-FacePerception_run-1_events.tsv") - tab = TabularInput(events, sidecar) - series = tab.series_filtered - - if form == "long": - df = series.copy() - convert_to_form(df, self.schema, "long_tag") - series = df - - if tile_to and tile_to > len(series): - reps = (tile_to // len(series)) + 1 - series = pd.Series(list(series) * reps).iloc[:tile_to].reset_index(drop=True) - - return series - - -# Quick self-test -if __name__ == "__main__": - gen = DataGenerator() - print(f"Schema tags available: {len(gen._tags)}") - print(f"Sample string (5 tags): {gen.make_string(5)}") - print(f"Sample string (10 tags, 2 groups, depth 2): {gen.make_string(10, 2, 2)}") - print(f"Sample string (5 tags, 3 repeats): {gen.make_string(5, repeats=3)}") - print(f"Real data rows: {len(gen.load_real_data())}") - print(f"Tiled to 500: {len(gen.load_real_data(tile_to=500))}") diff --git a/benchmarks/report.py b/benchmarks/report.py deleted file mode 100644 index e10bfaeb..00000000 --- a/benchmarks/report.py +++ /dev/null @@ -1,914 +0,0 @@ -"""Generate analysis report from benchmark results. - -Reads the latest JSON results file and produces: - - Console summary tables - - Matplotlib figures saved to benchmarks/figures/{stem}/ - - A Markdown report in benchmarks/results/ - -Usage:: - - python report.py # latest results - python report.py results/benchmark_20260407_120000.json # specific file -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -import matplotlib - -matplotlib.use("Agg") # must be set before importing pyplot -import matplotlib.pyplot as plt # noqa: E402 -import pandas as pd - -RESULTS_DIR = Path(__file__).parent / "results" -_FIGURES_BASE = Path(__file__).parent / "figures" -_FIGURES_BASE.mkdir(exist_ok=True) - -# Consistent colours per engine -ENGINE_COLORS = { - "Basic search": "#1f77b4", - "Object search": "#ff7f0e", - "String search": "#2ca02c", - "String search (lookup)": "#d62728", -} - -# Map legacy engine labels (from older JSON files) to current display names -_ENGINE_LABEL_MAP = { - "basic_search": "Basic search", - "QueryHandler": "Object search", - "QueryHandler_loop": "Object search", - "StringQueryHandler": "String search", - "StringQueryHandler_no_lookup": "String search", - "SQH_no_lookup": "String search", - "search_series": "String search", - "search_series_no_lookup": "String search", - "search_strings": "String search", - "StringQueryHandler_with_lookup": "String search (lookup)", - "SQH_with_lookup": "String search (lookup)", - "search_series_with_lookup": "String search (lookup)", -} - - -def _normalize_engine_labels(data): - """Remap legacy engine label strings to current display names in-place.""" - for section in ("single_string", "series", "factor_sweeps", "real_data"): - for record in data.get(section, []): - if "engine" in record: - record["engine"] = _ENGINE_LABEL_MAP.get(record["engine"], record["engine"]) - return data - - -def load_results(path=None): - """Load benchmark results from JSON.""" - if path is None: - files = sorted(RESULTS_DIR.glob("benchmark_*.json")) - if not files: - print("No results files found in", RESULTS_DIR) - sys.exit(1) - path = files[-1] - else: - path = Path(path) - print(f"Loading results from {path}") - data = json.loads(path.read_text(encoding="utf-8")) - _normalize_engine_labels(data) - return data, path.stem - - -# ====================================================================== -# Console summary -# ====================================================================== - - -def print_single_string_summary(data): - """Print a pivoted summary table of single-string results.""" - records = data.get("single_string", []) - if not records: - return - df = pd.DataFrame(records) - print("\n" + "=" * 80) - print("SINGLE-STRING BENCHMARK SUMMARY (median seconds)") - print("=" * 80) - pivot = df.pivot_table( - index=["config_label", "query_label"], - columns="engine", - values="total_time", - aggfunc="first", - ) - # Convert to milliseconds for readability - pivot_ms = pivot * 1000 - pd.set_option("display.float_format", "{:.4f}".format) - pd.set_option("display.max_columns", 20) - pd.set_option("display.width", 200) - print(pivot_ms.to_string()) - print() - - -def print_series_summary(data): - """Print series-level benchmark summary.""" - records = data.get("series", []) - if not records: - return - df = pd.DataFrame(records) - print("\n" + "=" * 80) - print("SERIES BENCHMARK SUMMARY (median seconds)") - print("=" * 80) - pivot = df.pivot_table( - index=["config_label", "query_label"], - columns="engine", - values="total_time", - aggfunc="first", - ) - pivot_ms = pivot * 1000 - print(pivot_ms.to_string()) - print() - - -def print_sweep_summary(data): - """Print factor sweep summary.""" - records = data.get("factor_sweeps", []) - if not records: - return - df = pd.DataFrame(records) - print("\n" + "=" * 80) - print("FACTOR SWEEP SUMMARY") - print("=" * 80) - for factor in df["factor"].unique(): - sub = df[df["factor"] == factor] - pivot = sub.pivot_table(index="level", columns="engine", values="time", aggfunc="first") - pivot_ms = pivot * 1000 - print(f"\n--- {factor} (ms) ---") - print(pivot_ms.to_string()) - - -def print_real_data_summary(data): - """Print real-data benchmark summary.""" - records = data.get("real_data", []) - if not records: - return - df = pd.DataFrame(records) - print("\n" + "=" * 80) - print(f"REAL DATA BENCHMARK ({records[0].get('n_rows', '?')} rows)") - print("=" * 80) - pivot = df.pivot_table(index="query_label", columns="engine", values="total_time", aggfunc="first") - pivot_ms = pivot * 1000 - print(pivot_ms.to_string()) - print() - - -# ====================================================================== -# Plots -# ====================================================================== - - -def _color(engine): - return ENGINE_COLORS.get(engine, "#333333") - - -_FACTOR_AXIS_LABELS = { - "tag_count": "Tag count", - "nesting_depth": "Nesting depth", - "repeated_tags": "Repeated tags", - "group_count": "Group count", - "series_size": "List size (rows)", - "query_complexity": "Query complexity", - "schema_lookup": "Schema lookup mode", - "string_form": "String form", - "compile_vs_search": "Phase", - "per_operation": "Operation", - "deep_nest_bare_term": "Nesting depth", - "deep_nest_two_and": "Nesting depth", - "deep_nest_group_match": "Nesting depth", - "deep_nest_exact_group": "Nesting depth", - "deep_nest_negation": "Nesting depth", -} -_FACTOR_TITLES = { - "tag_count": "Tag count sweep", - "nesting_depth": "Nesting depth sweep", - "repeated_tags": "Repeated tags sweep", - "group_count": "Group count sweep", - "series_size": "List size sweep", - "query_complexity": "Query complexity sweep", - "schema_lookup": "Schema lookup overhead", - "string_form": "String form sweep", - "compile_vs_search": "Compile vs. search cost", - "per_operation": "Per-operation sweep", - "deep_nest_bare_term": "Deep nesting: bare term", - "deep_nest_two_and": "Deep nesting: two-term AND", - "deep_nest_group_match": "Deep nesting: group match", - "deep_nest_exact_group": "Deep nesting: exact group", - "deep_nest_negation": "Deep nesting: negation", -} - - -def plot_factor_sweep(data, stem): - """One figure per factor sweep with engines as separate lines.""" - records = data.get("factor_sweeps", []) - if not records: - return - df = pd.DataFrame(records) - - for factor in df["factor"].unique(): - sub = df[df["factor"] == factor].copy() - xlabel = _FACTOR_AXIS_LABELS.get(factor, factor) - title = _FACTOR_TITLES.get(factor, f"Factor sweep: {factor}") - - fig, ax = plt.subplots(figsize=(8, 5)) - for engine in sub["engine"].unique(): - edf = sub[sub["engine"] == engine].sort_values("level") - ax.plot(range(len(edf)), edf["time"].values * 1000, marker="o", label=engine, color=_color(engine)) - ax.set_xticks(range(len(edf))) - ax.set_xticklabels(edf["level"].astype(str), rotation=45, ha="right") - - ax.set_xlabel(xlabel) - ax.set_ylabel("Time (ms)") - ax.set_title(title) - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3) - fig.tight_layout() - fig.savefig(_figures_dir(stem) / f"benchmark_sweep_{factor}.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_sweep_{factor}.png") - - -def plot_series_scaling(data, stem): - """Plot total time vs series size for each engine.""" - records = data.get("factor_sweeps", []) - if not records: - return - df = pd.DataFrame(records) - sub = df[df["factor"] == "series_size"] - if sub.empty: - return - - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - - # Total time - ax = axes[0] - for engine in sub["engine"].unique(): - edf = sub[sub["engine"] == engine].sort_values("level") - ax.plot(edf["level"], edf["time"] * 1000, marker="o", label=engine, color=_color(engine)) - ax.set_xlabel("List size (rows)") - ax.set_ylabel("Total time (ms)") - ax.set_title("List search: total time") - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3) - - # Per-row time - ax = axes[1] - for engine in sub["engine"].unique(): - edf = sub[sub["engine"] == engine].sort_values("level") - if "per_row" in edf.columns: - ax.plot(edf["level"], edf["per_row"] * 1000, marker="o", label=engine, color=_color(engine)) - ax.set_xlabel("List size (rows)") - ax.set_ylabel("Per-row time (ms)") - ax.set_title("List search: per-row amortized cost") - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3) - - fig.tight_layout() - fig.savefig(_figures_dir(stem) / "benchmark_series_scaling.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_series_scaling.png") - - -def plot_compile_vs_search(data, stem): - """Bar chart comparing compilation time to per-search time.""" - records = data.get("factor_sweeps", []) - if not records: - return - df = pd.DataFrame(records) - sub = df[df["factor"] == "compile_vs_search"] - if sub.empty: - return - - fig, ax = plt.subplots(figsize=(8, 5)) - engines = sub["engine"].unique() - levels = sub["level"].unique() # compile, search - x = range(len(engines)) - width = 0.35 - - for i, level in enumerate(levels): - vals = [] - for eng in engines: - row = sub[(sub["engine"] == eng) & (sub["level"] == level)] - vals.append(row["time"].values[0] * 1000 if len(row) else 0) - offset = (i - 0.5) * width - ax.bar([xi + offset for xi in x], vals, width, label=level) - - ax.set_xticks(x) - ax.set_xticklabels(engines, rotation=15) - ax.set_ylabel("Time (ms)") - ax.set_title("Compilation vs per-search cost") - ax.legend() - ax.grid(True, alpha=0.3, axis="y") - fig.tight_layout() - fig.savefig(_figures_dir(stem) / "benchmark_compile_vs_search.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_compile_vs_search.png") - - -def plot_schema_lookup(data, stem): - """Grouped bar chart comparing No lookup vs With lookup across query types.""" - records = data.get("factor_sweeps", []) - if not records: - return - df = pd.DataFrame(records) - sub = df[df["factor"] == "schema_lookup"] - if sub.empty: - return - - modes = ["No lookup", "With lookup"] - query_labels = list(sub["level"].unique()) - x = range(len(query_labels)) - width = 0.35 - colors = {"No lookup": "#d62728", "With lookup": "#2ca02c"} - - fig, ax = plt.subplots(figsize=(9, 5)) - for i, mode in enumerate(modes): - vals = [] - for ql in query_labels: - row = sub[(sub["engine"] == mode) & (sub["level"] == ql)] - vals.append(row["time"].values[0] * 1000 if len(row) else 0) - offset = (i - 0.5) * width - bars = ax.bar([xi + offset for xi in x], vals, width, label=mode, color=colors[mode], alpha=0.85) - # Annotate bars with match counts if available - if "matches" in sub.columns: - for bar, ql in zip(bars, query_labels, strict=False): - row = sub[(sub["engine"] == mode) & (sub["level"] == ql)] - if len(row) and "matches" in row.columns: - m = int(row["matches"].values[0]) - label = f"{m} match{'es' if m != 1 else ''}" if m > 0 else "no match" - ax.text( - bar.get_x() + bar.get_width() / 2, - bar.get_height() + 0.0005, - label, - ha="center", - va="bottom", - fontsize=7, - rotation=45, - ) - - ax.set_xticks(list(x)) - ax.set_xticklabels(query_labels, rotation=20, ha="right") - ax.set_ylabel("Time (ms)") - ax.set_title("Schema lookup: timing and matching behaviour") - ax.legend() - ax.grid(True, alpha=0.3, axis="y") - fig.tight_layout() - fig.savefig(_figures_dir(stem) / "benchmark_schema_lookup.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_schema_lookup.png") - - -def plot_query_complexity_heatmap(data, stem): - """Heatmap of query complexity vs engine (single-string results).""" - records = data.get("single_string", []) - if not records: - return - df = pd.DataFrame(records) - # Pick one config for clarity - config = df["config_label"].unique()[len(df["config_label"].unique()) // 2] - sub = df[df["config_label"] == config] - - pivot = sub.pivot_table(index="query_label", columns="engine", values="total_time", aggfunc="first") - pivot_ms = pivot * 1000 - - fig, ax = plt.subplots(figsize=(12, 6)) - im = ax.imshow(pivot_ms.values, aspect="auto", cmap="YlOrRd") - ax.set_xticks(range(len(pivot_ms.columns))) - ax.set_xticklabels(pivot_ms.columns, rotation=45, ha="right", fontsize=8) - ax.set_yticks(range(len(pivot_ms.index))) - ax.set_yticklabels(pivot_ms.index, fontsize=8) - ax.set_title(f"Query × Engine time (ms) — config: {config}") - fig.colorbar(im, ax=ax, label="Time (ms)") - - # Annotate cells - for i in range(len(pivot_ms.index)): - for j in range(len(pivot_ms.columns)): - val = pivot_ms.values[i, j] - if pd.notna(val): - ax.text( - j, - i, - f"{val:.2f}", - ha="center", - va="center", - fontsize=7, - color="white" if val > pivot_ms.values[pd.notna(pivot_ms.values)].mean() else "black", - ) - - fig.tight_layout() - fig.savefig(_figures_dir(stem) / "benchmark_query_heatmap.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_query_heatmap.png") - - -def plot_real_data(data, stem): - """Bar chart of real-data results.""" - records = data.get("real_data", []) - if not records: - return - df = pd.DataFrame(records) - - pivot = df.pivot_table(index="query_label", columns="engine", values="total_time", aggfunc="first") - pivot_ms = pivot * 1000 - - fig, ax = plt.subplots(figsize=(10, 5)) - pivot_ms.plot(kind="bar", ax=ax, color=[_color(c) for c in pivot_ms.columns]) - ax.set_ylabel("Total time (ms)") - ax.set_title(f"Real BIDS data ({records[0].get('n_rows', '?')} rows)") - ax.legend(fontsize=8) - ax.grid(True, alpha=0.3, axis="y") - plt.xticks(rotation=45, ha="right") - fig.tight_layout() - fig.savefig(_figures_dir(stem) / "benchmark_real_data.png", dpi=150) - plt.close(fig) - print(f" Saved figures/{stem}/benchmark_real_data.png") - - -# ====================================================================== -# Markdown report -# ====================================================================== - - -def _pivot_to_md(pivot_ms, float_fmt=".3f"): - """Convert a pandas pivot table (in ms) to a Markdown table string.""" - lines = [] - headers = [""] + [str(c) for c in pivot_ms.columns] - lines.append("| " + " | ".join(headers) + " |") - lines.append("| " + " | ".join(["---"] * len(headers)) + " |") - for idx, row in pivot_ms.iterrows(): - label = str(idx) if not isinstance(idx, tuple) else " / ".join(str(x) for x in idx) - cells = [label] - for v in row: - cells.append(f"{v:{float_fmt}}" if pd.notna(v) else "—") - lines.append("| " + " | ".join(cells) + " |") - return "\n".join(lines) - - -def _engine_summary_table(data): - """Build a comparison table of the three search engines.""" - return ( - "| Feature | Basic search | Object search | String search |\n" - "| --- | --- | --- | --- |\n" - "| Input type | `pd.Series[str]` | `HedString` objects | Raw strings (`str`) |\n" - "| Schema required | No | Yes | Optional (via `schema_lookup`) |\n" - "| Batch API | `find_matching(series, query)` | Manual loop | `string_search(strings, query)` |\n" - "| Boolean AND | `word1, word2` | `term1 && term2` | same as Object search |\n" - "| Boolean OR | — | `term1 || term2` | same as Object search |\n" - "| Negation | `~word` | `~term` | same as Object search |\n" - "| Exact group `{}` | — | `{term1, term2}` | same as Object search |\n" - "| Optional exact `{:}` | — | `{term1, term2:}` | same as Object search |\n" - "| Logical group `[]` | — | `[term1, term2]` | same as Object search |\n" - "| Wildcard `?/?? /???` | — | Yes | same as Object search |\n" - "| Descendant wildcard | `*` suffix | `*` suffix | same as Object search |\n" - '| Quoted exact match | — | `"Exact-tag"` | same as Object search |\n' - "| Implementation | Regex on text | Recursive tree on parsed nodes | Recursive tree on StringNode |\n" - ) - - -def _figures_dir(stem: str) -> Path: - """Return (and create) the per-run figures subdirectory.""" - d = _FIGURES_BASE / stem - d.mkdir(parents=True, exist_ok=True) - return d - - -def generate_markdown_report(data, stem): - """Write a comprehensive Markdown report with tables, plots, and analysis.""" - mode = "quick" if data.get("quick") else "full" - lines = [] - - def h1(t): - lines.extend([f"# {t}", ""]) - - def h2(t): - lines.extend([f"## {t}", ""]) - - def h3(t): - lines.extend([f"### {t}", ""]) - - def p(t): - lines.extend([t, ""]) - - def img(alt, path): - lines.extend([f"![{alt}]({path})", ""]) - - def table(md): - lines.extend([md, ""]) - - # ------------------------------------------------------------------ - # Title and overview - # ------------------------------------------------------------------ - h1("HED search benchmark report") - p(f"**Run:** {data.get('timestamp', 'unknown')} ") - p(f"**Mode:** {mode}") - - h2("Overview") - p("This report compares the performance of the three HED string search engines provided by the `hedtools` package:") - p( - "1. **basic_search** (`hed.models.basic_search.find_matching`) — regex-based pattern matching " - "that operates directly on a `pd.Series` of raw HED strings. No schema required. " - "Supports simple boolean AND (`@`), negation (`~`), wildcards (`*`), and parenthesised groups.\n" - "2. **QueryHandler** (`hed.models.query_handler.QueryHandler`) — full expression-tree search " - "that operates on parsed `HedString` objects. Requires a loaded HED schema. " - "Supports AND, OR, negation, exact groups `{}`, optional exact `{:}`, logical groups `[]`, " - "wildcard child `?`/`??`/`???`, descendant wildcards, and quoted exact matches.\n" - "3. **String search** (`hed.models.string_search.StringQueryHandler`) — lightweight " - "tree-based search that operates on raw strings via `StringNode` duck-typing. Schema is " - "optional (via `schema_lookup` dict for ancestor queries). Provides `string_search()` " - "convenience function for a plain `list[str]`. Same query syntax as Object search." - ) - - h3("Engine capability matrix") - table(_engine_summary_table(data)) - - # ------------------------------------------------------------------ - # Benchmark query suite - # ------------------------------------------------------------------ - h2("Benchmark query suite") - p( - "All 18 operations below are used across the benchmarks. " - "The **single-string** and **series** benchmarks use the 12-query core set (✓); " - "the **per-operation sweep** uses all 18 on a fixed structured string; " - "**nesting-depth sweeps** use the 5-query subset marked †." - ) - table( - "| Category | Label | Object search / String search query | Basic search query | Core | Depth |\n" - "| --- | --- | --- | --- | :---: | :---: |\n" - "| Simple | `bare_term` | `Event` | `@Event` | ✓ | † |\n" - '| Simple | `exact_quoted` | `"Event"` (quoted exact match) | — unsupported | ✓ | |\n' - "| Simple | `wildcard_prefix` | `Def/*` | `Def/*` | ✓ | |\n" - "| Boolean | `and_2` | `Event && Action` | `@Event, @Action` | ✓ | † |\n" - "| Boolean | `and_3` | `Event && Action && Agent` | `@Event, @Action, @Agent` | ✓ | |\n" - "| Boolean | `deep_and_chain` | `Event && Action && Agent && Item && Red` | `@Event, @Action, @Agent, @Item, @Red` | | |\n" - "| Boolean | `or` | `Event \\|\\| Action` | — unsupported | ✓ | |\n" - "| Boolean | `negation` | `~Event` | `~Event` | ✓ | † |\n" - "| Boolean | `double_negation` | `~(~Event)` | — unsupported | | |\n" - "| Boolean | `nested_or_and` | `(Event \\|\\| Sensory-event) && (Action \\|\\| Agent)` | — unsupported | | |\n" - "| Group structural | `group_nesting` | `[Event && Action]` | `(Event, Action)` | ✓ | † |\n" - "| Group structural | `exact_group` | `{Event && Action}` | — unsupported | ✓ | † |\n" - "| Group structural | `exact_group_optional` | `{Event && Action: Agent}` | — unsupported | ✓ | |\n" - "| Group structural | `wildcard_?` | `{Event, ?}` | — unsupported | ✓ | |\n" - "| Group structural | `wildcard_??` | `{Event, ??}` | — unsupported | | |\n" - "| Group structural | `wildcard_???` | `{Event, ???}` | — unsupported | | |\n" - "| Complex | `descendant_nested` | `[Def && Onset]` | — unsupported | | |\n" - "| Complex | `complex_composite` | `{(Onset \\|\\| Offset), (Def \\|\\| {Def-expand}): ???}` | — unsupported | ✓ | |\n" - ) - - # ------------------------------------------------------------------ - # Key findings (populated from data) - # ------------------------------------------------------------------ - h2("Key findings") - findings = [] - - # Series speed — use series_size sweep so query and config are consistent; - # report ratio at the largest row count tested. - series_recs = data.get("series", []) - _sweep_recs = data.get("factor_sweeps", []) - if _sweep_recs: - swdf_series = pd.DataFrame(_sweep_recs) - ss = swdf_series[swdf_series["factor"] == "series_size"] - if not ss.empty: - max_level = ss["level"].max() - at_max = ss[ss["level"] == max_level] - bs_row = at_max[at_max["engine"] == "Basic search"]["time"] - qh_row = at_max[at_max["engine"] == "Object search"]["time"] - if not bs_row.empty and not qh_row.empty and bs_row.values[0] > 0: - ratio = qh_row.values[0] / bs_row.values[0] - findings.append( - f"**Batch throughput:** Basic search is ~{ratio:.0f}× faster than " - f"Object search in a row-by-row loop at {max_level:,} rows, " - f"because it leverages vectorised pandas `str.contains` regex matching." - ) - elif series_recs: - sdf = pd.DataFrame(series_recs) - # Group by engine + n_rows, then take the median across queries at each row count; - # report the ratio at the largest row count to avoid mixing incomparable workloads. - per_nrows = sdf.groupby(["engine", "n_rows"])["total_time"].median().reset_index() - max_nrows = per_nrows["n_rows"].max() - at_max = per_nrows[per_nrows["n_rows"] == max_nrows] - bs_row = at_max[at_max["engine"] == "Basic search"]["total_time"] - qh_row = at_max[at_max["engine"] == "Object search"]["total_time"] - if not bs_row.empty and not qh_row.empty and bs_row.values[0] > 0: - ratio = qh_row.values[0] / bs_row.values[0] - findings.append( - f"**Batch throughput:** Basic search is ~{ratio:.0f}× faster than " - f"Object search in a row-by-row loop at {max_nrows:,} rows, " - f"because it leverages vectorised pandas `str.contains` regex matching." - ) - - # SQH vs QH per string - single_recs = data.get("single_string", []) - if single_recs: - ssdf = pd.DataFrame(single_recs) - qh_avg = ssdf[ssdf["engine"] == "Object search"]["total_time"].mean() - sqh_avg = ssdf[ssdf["engine"] == "String search"]["total_time"].mean() - if qh_avg > 0 and sqh_avg > 0: - pct = (1 - sqh_avg / qh_avg) * 100 - findings.append( - f"**Single-string speed:** String search (no lookup) is ~{pct:.0f}% " - f"faster than Object search per string because it avoids schema-based " - f"`HedString` construction and uses lightweight string parsing." - ) - - # Schema lookup cost - sweeps = data.get("factor_sweeps", []) - if sweeps: - swdf = pd.DataFrame(sweeps) - lu = swdf[swdf["factor"] == "schema_lookup"] - if not lu.empty: - with_lu = lu[lu["level"] == "with_lookup"]["time"].mean() - no_lu = lu[lu["level"] == "no_lookup"]["time"].mean() - if no_lu > 0: - lu_pct = ((with_lu / no_lu) - 1) * 100 - if abs(lu_pct) < 5: - findings.append( - "**Schema-lookup overhead:** Enabling `schema_lookup` in " - "String search has negligible overhead for simple queries " - "(cost comes from queries that actually use ancestor matching)." - ) - else: - findings.append( - f"**Schema-lookup overhead:** Enabling `schema_lookup` in " - f"String search adds ~{lu_pct:.0f}% overhead for " - f"ancestor-based queries." - ) - - # Deep nesting - if sweeps: - nest_df = swdf[swdf["factor"] == "nesting_depth"] - if not nest_df.empty: - for eng in ["Object search", "String search (lookup)"]: - edf = nest_df[nest_df["engine"] == eng].sort_values("level") - if len(edf) >= 2: - t0 = edf.iloc[0]["time"] - t_last = edf.iloc[-1]["time"] - if t0 > 0: - ratio = t_last / t0 - findings.append( - f"**Nesting depth ({eng}):** At depth {edf.iloc[-1]['level']}, " - f"search time is ~{ratio:.1f}× the flat-string time." - ) - - # basic_search operation limitations - if sweeps: - po = swdf[swdf["factor"] == "per_operation"] - if not po.empty: - total = po["level"].nunique() - bs_supported = po[po["engine"] == "Basic search"]["level"].nunique() - unsupported = total - bs_supported - if unsupported > 0: - findings.append( - f"**Operation coverage:** Basic search supports " - f"{bs_supported} of {total} tested operations. " - f"The remaining {unsupported} operations (OR, exact groups, logical groups, " - f"wildcards `?`/`??`/`???`, quoted terms) require Object search or " - f"String search." - ) - - for f in findings: - p(f"- {f}") - - # ------------------------------------------------------------------ - # Single-string results - # ------------------------------------------------------------------ - if single_recs: - h2("Single-string performance") - p( - "Each of the 12 core queries (see Benchmark query suite above) was applied to a " - "single HED string of varying complexity. Times are medians of repeated runs, in milliseconds." - ) - ssdf = pd.DataFrame(single_recs) - pivot = ( - ssdf.pivot_table( - index=["config_label", "query_label"], columns="engine", values="total_time", aggfunc="first" - ) - * 1000 - ) - table(_pivot_to_md(pivot)) - - img("Query × Engine heatmap", f"../figures/{stem}/benchmark_query_heatmap.png") - - # ------------------------------------------------------------------ - # Series results - # ------------------------------------------------------------------ - if series_recs: - h2("Row-by-row search scaling") - p( - "Whole-list search: each engine processes all items in a list of strings for a " - "given query. Basic search uses vectorised regex on a `pd.Series`; String search uses " - "`StringQueryHandler.search()` per item on a plain list; Object search constructs a " - "`HedString` per row then searches. Times in milliseconds." - ) - sdf = pd.DataFrame(series_recs) - pivot = ( - sdf.pivot_table( - index=["config_label", "query_label"], columns="engine", values="total_time", aggfunc="first" - ) - * 1000 - ) - table(_pivot_to_md(pivot)) - - img("List search scaling", f"../figures/{stem}/benchmark_series_scaling.png") - - # ------------------------------------------------------------------ - # Factor sweeps - # ------------------------------------------------------------------ - h2("Factor sweeps") - p("Each sweep varies a single factor while holding others constant, measuring how performance degrades.") - - factor_descriptions = { - "tag_count": ( - "Number of tags in the HED string (1 to 100). Basic search time is dominated by " - "regex compilation overhead and stays roughly constant; tree-based engines scale " - "linearly with the number of nodes to traverse." - ), - "nesting_depth": ( - "Parenthesisation depth from 0 (flat) to 20. Deeper nesting increases the tree " - "walk for Object search and String search. Basic search sees variable cost because " - "deeper nesting means more delimiter positions for its cartesian-product verification." - ), - "repeated_tags": ( - "Repetitions of a target tag (0 to 40). Basic search's `verify_search_delimiters` " - "uses `itertools.product` over delimiter positions; repeated tags multiply the " - "search space. Tree-based engines are unaffected." - ), - "group_count": ( - "Number of parenthesised groups (1 to 20). More groups mean more children at the " - "top level for tree traversal." - ), - "series_size": ( - "Number of strings in the list (10 to 5000). basic_search scales sub-linearly " - "thanks to vectorised pandas regex applied to a `pd.Series`. All other engines " - "scale linearly (fixed per-item cost)." - ), - "query_complexity": ( - "Query expression complexity from a bare term to a multi-clause composite. " - "More clauses = more expression-tree nodes to evaluate per candidate." - ), - "schema_lookup": ( - "The `schema_lookup` dict (produced by `generate_schema_lookup(schema)`) controls " - "whether string search resolves parent-class queries. Without it, bare terms match " - "only exact tag names — `Event` does **not** match `Sensory-event`. With it, every " - "tag carries its full ancestor path, so `Event` matches any descendant. " - "The table shows timing (ms) and match count on a fixed short-form string " - "containing known Event and Action descendants." - ), - "string_form": ( - "Short-form vs long-form HED strings. Long-form strings have fully expanded " - "paths (e.g. `Event/Sensory-event`) and are longer, increasing regex and parse cost." - ), - "compile_vs_search": ( - "Decomposition of one-time query compilation cost vs per-string search cost. " - "Compilation is cheap for both engines; the per-search cost dominates." - ), - "per_operation": ( - "Individual operation types tested in isolation. Shows which operations are " - "expensive for each engine. basic_search shows NaN/— for unsupported operations." - ), - } - - # Deep nesting sub-sweeps - for rec in sweeps: - factor = rec["factor"] - if factor.startswith("deep_nest_") and factor not in factor_descriptions: - query_type = factor.replace("deep_nest_", "").replace("_", " ") - factor_descriptions[factor] = ( - f"Deep nesting sweep for *{query_type}* queries at depths 1–20. " - f"Shows how nesting interacts with specific query patterns." - ) - - factors = sorted({rec["factor"] for rec in sweeps}) - for factor in factors: - h3(factor.replace("_", " ").title()) - desc = factor_descriptions.get(factor, "") - if desc: - p(desc) - - sub = pd.DataFrame([r for r in sweeps if r["factor"] == factor]) - - if factor == "schema_lookup": - # Build an expanded table showing both time (ms) and match count side by side. - modes = ["No lookup", "With lookup"] - headers = ["Query"] + [f"{m}: time (ms)" for m in modes] + [f"{m}: matches" for m in modes] - lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join(["---"] * len(headers)) + " |"] - for ql in sub["level"].unique(): - row_cells = [ql] - for m in modes: - r = sub[(sub["engine"] == m) & (sub["level"] == ql)] - row_cells.append(f"{r['time'].values[0] * 1000:.3f}" if len(r) else "—") - for m in modes: - r = sub[(sub["engine"] == m) & (sub["level"] == ql)] - mc = int(r["matches"].values[0]) if len(r) and "matches" in r.columns else "—" - row_cells.append(str(mc)) - lines.append("| " + " | ".join(row_cells) + " |") - table("\n".join(lines)) - img("Schema lookup: timing and matching behaviour", f"../figures/{stem}/benchmark_schema_lookup.png") - else: - # Inline table for this factor - pivot = sub.pivot_table(index="level", columns="engine", values="time", aggfunc="first") * 1000 - table(_pivot_to_md(pivot)) - img(factor, f"../figures/{stem}/benchmark_sweep_{factor}.png") - - # ------------------------------------------------------------------ - # Real data - # ------------------------------------------------------------------ - real_recs = data.get("real_data", []) - if real_recs: - h2("Real BIDS data") - n_rows = real_recs[0].get("n_rows", "?") - p( - f"Search over {n_rows} rows of real BIDS event data " - f"(`eeg_ds003645s_hed` test dataset, HED_column values). " - f"Times in milliseconds." - ) - rdf = pd.DataFrame(real_recs) - pivot = rdf.pivot_table(index="query_label", columns="engine", values="total_time", aggfunc="first") * 1000 - table(_pivot_to_md(pivot)) - img("Real BIDS data", f"../figures/{stem}/benchmark_real_data.png") - - # ------------------------------------------------------------------ - # Recommendations - # ------------------------------------------------------------------ - h2("Recommendations") - p( - "**Choose Basic search when:** You need the fastest possible batch search over a " - "`pd.Series`, your queries use only simple terms, AND, negation, or descendant wildcards (`*`), " - "and you don't need schema-aware matching. Ideal for filtering event files where " - "speed matters and queries are simple." - ) - p( - "**Choose String search when:** You need the full query language (OR, exact " - "groups, logical groups, wildcards) but want to avoid the overhead of parsing every " - "HED string through the schema. `string_search()` is the best general-purpose " - "option when operating on raw strings from tabular files." - ) - p( - "**Choose Object search when:** You already have parsed `HedString` objects (e.g. " - "from validation pipelines), or you need exact schema-validated matching. The " - "additional overhead comes from `HedString` construction, not the search itself." - ) - - # ------------------------------------------------------------------ - # Methodology - # ------------------------------------------------------------------ - h2("Methodology") - p( - f"- **Timing:** `timeit` with {20 if not data.get('quick') else 10} iterations " - f"(single-string), {5 if not data.get('quick') else 3} iterations (list search), " - f"{10 if not data.get('quick') else 5} iterations (sweeps). Median of all iterations reported.\n" - f"- **Schema:** HED 8.4.0 loaded once and reused across all benchmarks.\n" - f"- **Data generation:** Synthetic strings built from real schema tags with controlled " - f"tag count, nesting depth, group count, and tag repetition.\n" - f"- **schema_lookup:** Generated via `generate_schema_lookup(schema)` — a dict mapping " - f"each short tag to its ancestor tuple.\n" - f"- **Environment:** Results depend on hardware; relative ratios between engines are " - f"the meaningful comparison." - ) - - # Write - report_path = RESULTS_DIR / f"{stem}_report.md" - report_path.write_text("\n".join(lines), encoding="utf-8") - print(f" Saved {report_path}") - - -# ====================================================================== -# Main -# ====================================================================== - - -def main(path=None): - data, stem = load_results(path) - - # Console summaries - print_single_string_summary(data) - print_series_summary(data) - print_sweep_summary(data) - print_real_data_summary(data) - - # Plots - print("\nGenerating plots…") - plot_factor_sweep(data, stem) - plot_series_scaling(data, stem) - plot_compile_vs_search(data, stem) - plot_schema_lookup(data, stem) - plot_query_complexity_heatmap(data, stem) - plot_real_data(data, stem) - - # Markdown - print("\nGenerating Markdown report…") - generate_markdown_report(data, stem) - - print("\nDone.") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate benchmark report") - parser.add_argument("results_file", nargs="?", default=None, help="Path to results JSON") - args = parser.parse_args() - main(args.results_file) diff --git a/benchmarks/search_benchmark.py b/benchmarks/search_benchmark.py deleted file mode 100644 index aad0084d..00000000 --- a/benchmarks/search_benchmark.py +++ /dev/null @@ -1,772 +0,0 @@ -"""HED search performance benchmark harness. - -Measures compilation time, single-string search time, and series search time -for all three HED search engines across a matrix of query types + data configs. - -Usage:: - - python search_benchmark.py # full benchmark - python search_benchmark.py --quick # fast smoke-test -""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -import timeit -import tracemalloc -from datetime import datetime -from pathlib import Path - -import pandas as pd - -# Ensure the repo root is importable when running the script directly -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from data_generator import DataGenerator # noqa: E402 - -from hed import HedString, QueryHandler # noqa: E402 -from hed.models.basic_search import find_matching # noqa: E402 -from hed.models.string_search import StringQueryHandler, string_search # noqa: E402 - -RESULTS_DIR = Path(__file__).parent / "results" -RESULTS_DIR.mkdir(exist_ok=True) - - -# ====================================================================== -# Timing helpers -# ====================================================================== - - -def time_it(func, n_runs=5): - """Return (median_seconds, all_times) for calling *func* n_runs times.""" - times = [] - for _ in range(n_runs): - t = timeit.timeit(func, number=1) - times.append(t) - times.sort() - median = times[len(times) // 2] - return median, times - - -def measure_memory(func): - """Return peak memory (bytes) used by *func*.""" - tracemalloc.start() - func() - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - return peak - - -# ====================================================================== -# Query definitions — (label, basic_search_query, qh_query) -# basic_search_query = None means "not supported by basic_search" -# ====================================================================== - -QUERIES = [ - # --- Simple terms --- - ("single_bare_term", "@Event", "Event"), - ("single_exact_term", None, '"Event"'), - ("single_wildcard", "Def/*", "Def/*"), - # --- Boolean --- - ("two_term_and", "@Event, @Action", "Event && Action"), - ("two_term_or", None, "Event || Action"), - ("negation", "~Event", "~Event"), - # --- Groups --- - ("group_nesting", "(Event, Action)", "[Event && Action]"), - ("exact_group", None, "{Event && Action}"), - ("exact_group_optional", None, "{Event && Action: Agent}"), - ("wildcard_child", None, "{Event, ?}"), - # --- Complex --- - ("three_term_and", "@Event, @Action, @Agent", "Event && Action && Agent"), - ("complex_composite", None, "{(Onset || Offset), (Def || {Def-expand}): ???}"), -] - - -# ====================================================================== -# Single-string benchmarks -# ====================================================================== - - -class SingleStringBenchmark: - """Benchmark each engine on a single HED string.""" - - def __init__(self, gen: DataGenerator, n_runs=20): - self.gen = gen - self.schema = gen.schema - self.lookup = gen.lookup - self.n_runs = n_runs - - def run_all(self, string_configs): - """Run all queries against all string configurations. - - Parameters: - string_configs: list of dicts with keys matching DataGenerator.make_string params - plus a 'label' key for identification. - - Returns: - list[dict]: One record per (query, config, engine) combination. - """ - records = [] - for cfg in string_configs: - label = cfg.pop("label") - raw = self.gen.make_string(**cfg) - cfg["label"] = label # restore - - for q_label, bs_query, qh_query in QUERIES: - # --- basic_search --- - if bs_query is not None: - rec = self._bench_basic(raw, bs_query, label, q_label) - records.append(rec) - - # --- QueryHandler --- - rec = self._bench_query_handler(raw, qh_query, label, q_label) - records.append(rec) - - # --- StringQueryHandler (no lookup) --- - rec = self._bench_string_qh(raw, qh_query, label, q_label, schema_lookup=None, suffix="no_lookup") - records.append(rec) - - # --- StringQueryHandler (with lookup) --- - rec = self._bench_string_qh( - raw, qh_query, label, q_label, schema_lookup=self.lookup, suffix="with_lookup" - ) - records.append(rec) - - return records - - def _bench_basic(self, raw, query, cfg_label, q_label): - series = pd.Series([raw]) - # Compilation (regex build is inside find_matching, not separable easily) - med, _ = time_it(lambda: find_matching(series, query), self.n_runs) - matches = int(find_matching(series, query).sum()) - return { - "engine": "Basic search", - "query_label": q_label, - "config_label": cfg_label, - "query": query, - "compile_time": None, # not separable - "search_time": med, - "total_time": med, - "matches": matches, - } - - def _bench_query_handler(self, raw, query, cfg_label, q_label): - # Compilation - comp_med, _ = time_it(lambda: QueryHandler(query), self.n_runs) - qh = QueryHandler(query) - - # Need to parse HedString each time (part of the cost) - def do_search(): - hs = HedString(raw, self.schema) - return qh.search(hs) - - search_med, _ = time_it(do_search, self.n_runs) - result = do_search() - return { - "engine": "Object search", - "query_label": q_label, - "config_label": cfg_label, - "query": query, - "compile_time": comp_med, - "search_time": search_med, - "total_time": comp_med + search_med, - "matches": len(result), - } - - def _bench_string_qh(self, raw, query, cfg_label, q_label, schema_lookup, suffix): - comp_med, _ = time_it(lambda: StringQueryHandler(query), self.n_runs) - sqh = StringQueryHandler(query) - search_med, _ = time_it(lambda: sqh.search(raw, schema_lookup=schema_lookup), self.n_runs) - result = sqh.search(raw, schema_lookup=schema_lookup) - label = "String search" if suffix == "no_lookup" else "String search (lookup)" - return { - "engine": label, - "query_label": q_label, - "config_label": cfg_label, - "query": query, - "compile_time": comp_med, - "search_time": search_med, - "total_time": comp_med + search_med, - "matches": len(result), - } - - -# ====================================================================== -# Series benchmarks -# ====================================================================== - - -class SeriesBenchmark: - """Benchmark each engine on a pd.Series of HED strings.""" - - def __init__(self, gen: DataGenerator, n_runs=5): - self.gen = gen - self.schema = gen.schema - self.lookup = gen.lookup - self.n_runs = n_runs - - def run_all(self, series_configs): - """Run selected queries against series of varying size. - - Parameters: - series_configs: list of dicts with keys 'label', 'n_rows', plus - DataGenerator.make_series params. - - Returns: - list[dict]: One record per (query, config, engine) combination. - """ - records = [] - for cfg in series_configs: - label = cfg.pop("label") - n_rows = cfg["n_rows"] - series = self.gen.make_series(**cfg) - cfg["label"] = label # restore - - # Use a subset of queries for series (too slow to run all × all) - # For small series test all; for large ones test representative subset - queries_to_test = QUERIES if n_rows <= 500 else QUERIES[:6] - for q_label, bs_query, qh_query in queries_to_test: - print(f" Series {label} | {q_label} ({n_rows} rows)…") - - # --- basic_search --- - if bs_query is not None: - rec = self._bench_basic_series(series, bs_query, label, q_label, n_rows) - records.append(rec) - - # --- String search (StringQueryHandler) no lookup --- - rec = self._bench_search_series(series, qh_query, label, q_label, n_rows, None, "no_lookup") - records.append(rec) - - # --- String search (StringQueryHandler) with lookup --- - rec = self._bench_search_series(series, qh_query, label, q_label, n_rows, self.lookup, "with_lookup") - records.append(rec) - - # --- Object search (QueryHandler loop) --- - rec = self._bench_qh_loop(series, qh_query, label, q_label, n_rows) - records.append(rec) - - return records - - def _bench_basic_series(self, series, query, cfg_label, q_label, n_rows): - med, _ = time_it(lambda: find_matching(series, query), self.n_runs) - matches = int(find_matching(series, query).sum()) - return { - "engine": "Basic search", - "query_label": q_label, - "config_label": cfg_label, - "n_rows": n_rows, - "total_time": med, - "per_row": med / n_rows, - "matches": matches, - } - - def _bench_search_series(self, series, query, cfg_label, q_label, n_rows, lookup, suffix): - strings = series.tolist() - med, _ = time_it(lambda: string_search(strings, query, schema_lookup=lookup), self.n_runs) - matches = sum(string_search(strings, query, schema_lookup=lookup)) - label = "String search" if suffix == "no_lookup" else "String search (lookup)" - return { - "engine": label, - "query_label": q_label, - "config_label": cfg_label, - "n_rows": n_rows, - "total_time": med, - "per_row": med / n_rows, - "matches": matches, - } - - def _bench_qh_loop(self, series, query, cfg_label, q_label, n_rows): - qh = QueryHandler(query) - schema = self.schema - - def do_all(): - for s in series: - if pd.notna(s) and s: - hs = HedString(s, schema) - qh.search(hs) - - med, _ = time_it(do_all, self.n_runs) - # count matches - count = 0 - for s in series: - if pd.notna(s) and s: - hs = HedString(s, schema) - if qh.search(hs): - count += 1 - return { - "engine": "Object search", - "query_label": q_label, - "config_label": cfg_label, - "n_rows": n_rows, - "total_time": med, - "per_row": med / n_rows, - "matches": count, - } - - -# ====================================================================== -# Factor sweeps -# ====================================================================== - - -class FactorSweep: - """Isolate the effect of one variable on performance.""" - - def __init__(self, gen: DataGenerator, n_runs=10): - self.gen = gen - self.schema = gen.schema - self.lookup = gen.lookup - self.n_runs = n_runs - - def sweep_tag_count(self, tag_counts=(1, 5, 10, 25, 50, 100)): - """Vary number of tags per string, fixed simple query.""" - query = "Event" - bs_query = "@Event" - records = [] - for nt in tag_counts: - raw = self.gen.make_string(n_tags=nt) - for engine, med in self._bench_all_engines(raw, query, bs_query): - records.append({"factor": "tag_count", "level": nt, "engine": engine, "time": med}) - return records - - def sweep_nesting_depth(self, depths=(0, 1, 2, 3, 5, 10, 15, 20)): - """Vary nesting depth using deeply nested strings.""" - query = "Event" - bs_query = "@Event" - records = [] - for d in depths: - if d == 0: - raw = self.gen.make_string(n_tags=10) - else: - raw = self.gen.make_deeply_nested_string(depth=d, tags_per_level=2) - for engine, med in self._bench_all_engines(raw, query, bs_query): - records.append({"factor": "nesting_depth", "level": d, "engine": engine, "time": med}) - return records - - def sweep_repeated_tags(self, repeat_counts=(0, 3, 5, 10, 20, 40)): - """Vary duplicate tag count — stresses basic_search cartesian product. - - Uses strings that actually contain 'Event' and 'Action' as the repeated - tags so the group query ``(Event, Action)`` triggers combinatorial matching. - """ - query = "(Event, Action)" - bs_query = "(Event, Action)" - records = [] - for r in repeat_counts: - raw = self.gen.make_string_with_specific_tags( - ["Event", "Action"], n_extra=3, n_groups=1, depth=1, repeats=r - ) - for engine, med in self._bench_all_engines(raw, query, bs_query): - records.append({"factor": "repeated_tags", "level": r, "engine": engine, "time": med}) - return records - - def sweep_group_count(self, group_counts=(0, 1, 5, 10, 20)): - """Vary number of groups per string.""" - query = "Event" - bs_query = "@Event" - records = [] - for ng in group_counts: - raw = self.gen.make_string(n_tags=max(10, ng * 2 + 3), n_groups=ng, depth=1) - for engine, med in self._bench_all_engines(raw, query, bs_query): - records.append({"factor": "group_count", "level": ng, "engine": engine, "time": med}) - return records - - def sweep_series_size(self, sizes=(10, 100, 500, 1000, 5000)): - """Vary series length.""" - query = "Event" - bs_query = "@Event" - records = [] - for n in sizes: - series = self.gen.make_series(n_rows=n, n_tags=10, n_groups=2, depth=1) - for engine, med in self._bench_series_engines(series, query, bs_query, n): - records.append({"factor": "series_size", "level": n, "engine": engine, "time": med, "per_row": med / n}) - return records - - def sweep_query_complexity(self): - """Compare queries of increasing complexity.""" - raw = self.gen.make_string(n_tags=20, n_groups=5, depth=2) - complexity_queries = [ - ("1_single_term", "@Event", "Event"), - ("2_two_and", "@Event, @Action", "Event && Action"), - ("3_three_and", "@Event, @Action, @Agent", "Event && Action && Agent"), - ("4_or", None, "Event || Action"), - ("5_negation", "~Event", "~Event"), - ("6_group", "(Event, Action)", "[Event && Action]"), - ("7_exact", None, "{Event && Action}"), - ("8_complex", None, "{(Onset || Offset), (Def || {Def-expand}): ???}"), - ] - records = [] - for clabel, bs_q, qh_q in complexity_queries: - for engine, med in self._bench_all_engines(raw, qh_q, bs_q): - records.append({"factor": "query_complexity", "level": clabel, "engine": engine, "time": med}) - return records - - def sweep_schema_lookup(self): - """Compare StringQueryHandler with vs without schema_lookup across query types. - - Uses a fixed short-form string containing known descendants of Event and Action so - the behavioural difference (which strings match) is deterministic. - """ - # Fixed short-form string with known Event and Action descendants. - # Sensory-event, Agent-action, Data-feature are Event descendants; - # Communicate, Clap-hands are Action descendants. - raw = ( - "Sensory-event, Agent-action, Data-feature, Communicate, Clap-hands, " - "Communicate-gesturally, Blue, High, (Red, Move), (Experiment-control, Frown)" - ) - queries = [ - ("Ancestor: Event", "Event"), - ("Ancestor: Action", "Action"), - ("Exact: Sensory-event", "Sensory-event"), - ("Compound: Event && Action", "Event && Action"), - ] - records = [] - for q_label, query in queries: - sqh = StringQueryHandler(query) - for with_lookup in [False, True]: - lk = self.lookup if with_lookup else None - mode = "With lookup" if with_lookup else "No lookup" - med, _ = time_it(lambda lk=lk, _sqh=sqh: _sqh.search(raw, schema_lookup=lk), self.n_runs) - matches = len(sqh.search(raw, schema_lookup=lk)) - records.append( - {"factor": "schema_lookup", "level": q_label, "engine": mode, "time": med, "matches": matches} - ) - return records - - def sweep_string_form(self): - """Compare short vs long form strings.""" - query = "Event" - bs_query = "@Event" - records = [] - for form in ["short", "long"]: - raw = self.gen.make_string(n_tags=15, n_groups=3, depth=1, form=form) - for engine, med in self._bench_all_engines(raw, query, bs_query): - records.append({"factor": "string_form", "level": form, "engine": engine, "time": med}) - return records - - def sweep_compilation_vs_search(self): - """Separate compilation cost from per-search cost.""" - raw = self.gen.make_string(n_tags=15, n_groups=3, depth=1) - query = "Event" - records = [] - - # QueryHandler - comp, _ = time_it(lambda: QueryHandler(query), self.n_runs) - qh = QueryHandler(query) - - def qh_search(): - hs = HedString(raw, self.schema) - qh.search(hs) - - search_med, _ = time_it(qh_search, self.n_runs) - records.append({"factor": "compile_vs_search", "level": "compile", "engine": "Object search", "time": comp}) - records.append( - {"factor": "compile_vs_search", "level": "search", "engine": "Object search", "time": search_med} - ) - - # StringQueryHandler - comp2, _ = time_it(lambda: StringQueryHandler(query), self.n_runs) - sqh = StringQueryHandler(query) - search_med2, _ = time_it(lambda: sqh.search(raw, schema_lookup=self.lookup), self.n_runs) - records.append({"factor": "compile_vs_search", "level": "compile", "engine": "String search", "time": comp2}) - records.append( - {"factor": "compile_vs_search", "level": "search", "engine": "String search", "time": search_med2} - ) - - return records - - def sweep_per_operation(self): - """Test every query operation type on the same string. - - Uses a string with enough structure to exercise all operations: - groups, nested groups, Def tags, Onset, etc. - """ - # Build a string with structure that can match all query types - raw = ( - "Sensory-event, Action, Agent, " - "(Event, (Onset, (Def/MyDef))), " - "(Offset, Item, (Def-expand/MyDef, (Red, Blue))), " - "(Visual-presentation, Square, Green)" - ) - - operation_queries = [ - # (label, basic_search_query, qh_query) - ("bare_term", "@Event", "Event"), - ("exact_quoted", None, '"Sensory-event"'), - ("wildcard_prefix", "Def/*", "Def/*"), - ("and_2", "@Event, @Action", "Event && Action"), - ("and_3", "@Event, @Action, @Agent", "Event && Action && Agent"), - ("or", None, "Event || Action"), - ("negation", "~Event", "~Event"), - ("nested_group_[]", "(Event, Action)", "[Event && Action]"), - ("exact_group_{}", None, "{Event && Action}"), - ("exact_optional_{:}", None, "{Event && Action: Agent}"), - ("wildcard_?", None, "{Event, ?}"), - ("wildcard_??", None, "{Event, ??}"), - ("wildcard_???", None, "{Event, ???}"), - ("descendant_nested", None, "[Def && Onset]"), - ("complex_onset_def", None, "{(Onset || Offset), (Def || {Def-expand}): ???}"), - ("deep_and_chain", "@Event, @Action, @Agent, @Item, @Red", "Event && Action && Agent && Item && Red"), - ("nested_or_and", None, "(Event || Sensory-event) && (Action || Agent)"), - ("double_negation", None, "~(~Event)"), - ] - - records = [] - for op_label, bs_q, qh_q in operation_queries: - for engine, med in self._bench_all_engines(raw, qh_q, bs_q): - records.append({"factor": "per_operation", "level": op_label, "engine": engine, "time": med}) - return records - - def sweep_deep_nesting_by_query(self): - """Test how different query types perform on deeply nested strings.""" - depths = [1, 5, 10, 20] - queries = [ - ("bare_term", "@Event", "Event"), - ("two_and", "@Event, @Action", "Event && Action"), - ("group_match", "(Event, Action)", "[Event && Action]"), - ("exact_group", None, "{Event && Action}"), - ("negation", "~Event", "~Event"), - ] - records = [] - for d in depths: - raw = self.gen.make_deeply_nested_string(depth=d, tags_per_level=2) - for q_label, bs_q, qh_q in queries: - for engine, med in self._bench_all_engines(raw, qh_q, bs_q): - records.append( - { - "factor": f"deep_nest_{q_label}", - "level": d, - "engine": engine, - "time": med, - } - ) - return records - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _bench_all_engines(self, raw, qh_query, bs_query=None): - """Yield (engine_name, median_time) for all engines on a single string.""" - series1 = pd.Series([raw]) - - # basic_search - if bs_query is not None: - med, _ = time_it(lambda: find_matching(series1, bs_query), self.n_runs) - yield "Basic search", med - - # QueryHandler - qh = QueryHandler(qh_query) - - def qh_search(): - hs = HedString(raw, self.schema) - qh.search(hs) - - med, _ = time_it(qh_search, self.n_runs) - yield "Object search", med - - # StringQueryHandler no lookup - sqh = StringQueryHandler(qh_query) - med, _ = time_it(lambda: sqh.search(raw, schema_lookup=None), self.n_runs) - yield "String search", med - - # StringQueryHandler with lookup - med, _ = time_it(lambda: sqh.search(raw, schema_lookup=self.lookup), self.n_runs) - yield "String search (lookup)", med - - def _bench_series_engines(self, series, qh_query, bs_query, n_rows): - """Yield (engine_name, median_time) for series-level engines.""" - # basic_search - if bs_query is not None: - med, _ = time_it(lambda: find_matching(series, bs_query), self.n_runs) - yield "Basic search", med - - # String search no lookup - strings = series.tolist() - med, _ = time_it(lambda: string_search(strings, qh_query, schema_lookup=None), self.n_runs) - yield "String search", med - - # String search with lookup - med, _ = time_it(lambda: string_search(strings, qh_query, schema_lookup=self.lookup), self.n_runs) - yield "String search (lookup)", med - - # QueryHandler loop - qh = QueryHandler(qh_query) - schema = self.schema - - def qh_loop(): - for s in series: - if pd.notna(s) and s: - hs = HedString(s, schema) - qh.search(hs) - - med, _ = time_it(qh_loop, self.n_runs) - yield "Object search", med - - -# ====================================================================== -# Main orchestrator -# ====================================================================== - - -def run_full_benchmark(quick=False): - """Run the complete benchmark suite and save results.""" - print("Initialising DataGenerator (loading schema)…") - gen = DataGenerator() - - n_single = 10 if quick else 20 - n_series = 3 if quick else 10 - n_sweep = 5 if quick else 10 - - # ------------------------------------------------------------------ - # 1. Single-string benchmark - # ------------------------------------------------------------------ - print("\n=== Single-string benchmarks ===") - ssb = SingleStringBenchmark(gen, n_runs=n_single) - - string_configs = [ - {"label": "tiny_1tag", "n_tags": 1}, - {"label": "small_5tag", "n_tags": 5}, - {"label": "medium_10tag", "n_tags": 10, "n_groups": 2, "depth": 1}, - {"label": "large_25tag", "n_tags": 25, "n_groups": 5, "depth": 2}, - {"label": "xlarge_50tag", "n_tags": 50, "n_groups": 10, "depth": 2}, - ] - if not quick: - string_configs.append({"label": "xxlarge_100tag", "n_tags": 100, "n_groups": 15, "depth": 3}) - single_results = ssb.run_all(string_configs) - print(f" Collected {len(single_results)} single-string records.") - - # ------------------------------------------------------------------ - # 2. Series benchmark - # ------------------------------------------------------------------ - print("\n=== Series benchmarks ===") - sb = SeriesBenchmark(gen, n_runs=n_series) - - if quick: - series_sizes = [10, 100, 500] - else: - series_sizes = [10, 100, 500, 1000, 5000] - - series_configs = [] - for n in series_sizes: - series_configs.append({"label": f"homo_{n}", "n_rows": n, "n_tags": 10, "n_groups": 2, "depth": 1}) - for n in [100, 1000] if not quick else [100]: - series_configs.append({"label": f"hetero_{n}", "n_rows": n, "n_tags": 10, "heterogeneous": True}) - - series_results = sb.run_all(series_configs) - print(f" Collected {len(series_results)} series records.") - - # ------------------------------------------------------------------ - # 3. Factor sweeps - # ------------------------------------------------------------------ - print("\n=== Factor sweeps ===") - fs = FactorSweep(gen, n_runs=n_sweep) - - sweep_results = [] - for name, method in [ - ("tag_count", fs.sweep_tag_count), - ("nesting_depth", fs.sweep_nesting_depth), - ("repeated_tags", fs.sweep_repeated_tags), - ("group_count", fs.sweep_group_count), - ("series_size", fs.sweep_series_size), - ("query_complexity", fs.sweep_query_complexity), - ("schema_lookup", fs.sweep_schema_lookup), - ("string_form", fs.sweep_string_form), - ("compile_vs_search", fs.sweep_compilation_vs_search), - ("per_operation", fs.sweep_per_operation), - ("deep_nesting_by_query", fs.sweep_deep_nesting_by_query), - ]: - print(f" Sweep: {name}") - sweep_results.extend(method()) - - print(f" Collected {len(sweep_results)} sweep records.") - - # ------------------------------------------------------------------ - # 4. Real data benchmark - # ------------------------------------------------------------------ - print("\n=== Real data benchmark ===") - real_series = gen.load_real_data() - real_n = len(real_series) - print(f" Real data: {real_n} rows") - - real_results = [] - for q_label, bs_query, qh_query in QUERIES: - if bs_query is not None: - med, _ = time_it(lambda bs_query=bs_query: find_matching(real_series, bs_query), n_series) - real_results.append( - { - "engine": "Basic search", - "query_label": q_label, - "total_time": med, - "per_row": med / real_n, - "n_rows": real_n, - } - ) - - real_strings = real_series.tolist() - med, _ = time_it( - lambda qh_query=qh_query, _rs=real_strings: string_search(_rs, qh_query, schema_lookup=gen.lookup), - n_series, - ) - real_results.append( - { - "engine": "String search", - "query_label": q_label, - "total_time": med, - "per_row": med / real_n, - "n_rows": real_n, - } - ) - - qh = QueryHandler(qh_query) - schema = gen.schema - - def qh_loop(qh=qh, schema=schema): - for s in real_series: - if pd.notna(s) and s: - hs = HedString(s, schema) - qh.search(hs) - - med, _ = time_it(qh_loop, n_series) - real_results.append( - { - "engine": "Object search", - "query_label": q_label, - "total_time": med, - "per_row": med / real_n, - "n_rows": real_n, - } - ) - - print(f" Collected {len(real_results)} real-data records.") - - # ------------------------------------------------------------------ - # Save - # ------------------------------------------------------------------ - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output = { - "timestamp": timestamp, - "quick": quick, - "single_string": single_results, - "series": series_results, - "factor_sweeps": sweep_results, - "real_data": real_results, - } - out_path = RESULTS_DIR / f"benchmark_{timestamp}.json" - out_path.write_text(json.dumps(output, indent=2, default=str), encoding="utf-8") - print(f"\nResults saved to {out_path}") - return output - - -# ====================================================================== -# Entry point -# ====================================================================== - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="HED search benchmark") - parser.add_argument("--quick", action="store_true", help="Reduced run for smoke testing") - args = parser.parse_args() - run_full_benchmark(quick=args.quick)