Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -559,3 +559,28 @@ jobs:
cd /tmp
/tmp/smoke-sdist/bin/python -c "import fastapi_cache, importlib.metadata, pathlib; p=str(pathlib.Path(fastapi_cache.__file__).resolve()); assert 'site-packages' in p, p; assert importlib.metadata.version('fastapi-cache-stable'); print(p)"
/tmp/smoke-sdist/bin/pip check

bounded-inmemory-scalability:
name: Bounded InMemory 10k/100k write scaling (Python ${{ matrix.python }})
runs-on: ubuntu-24.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
python: ["3.8", "3.12"]
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: ${{ matrix.python }}
- name: Install project with bounded runtime profile
run: |
python -m pip install --upgrade pip
if [ "${{ matrix.python }}" = "3.8" ]; then
python -m pip install -e . "fastapi==0.111.1" "pydantic==1.10.27"
else
python -m pip install -e .
fi
python -m pip check
- name: Enforce sublinear 10k/100k write scaling
run: python scripts/benchmark_bounded_inmemory.py --assert-scaling
1 change: 1 addition & 0 deletions changelog.d/54.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add an optional `InMemoryBackend(max_entries=...)` capacity limit with per-instance least-recently-used eviction, expired-entry cleanup before eviction, and no behavior change when omitted. The bound counts entries rather than payload bytes and does not synchronize separate worker processes.
135 changes: 130 additions & 5 deletions fastapi_cache/backends/inmemory.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
"""In-process cache with optional bounded, per-instance LRU capacity.

Unbounded default preserves historical lazy-expiration semantics.
Bounded writes use an indexed min-heap for expiry and an OrderedDict
for recency: O(log capacity), no full-store scans on ordinary writes.
"""

import time
from asyncio import Lock
from dataclasses import dataclass
from typing import Dict, Optional, Tuple
from typing import Dict, List, Optional, OrderedDict, Tuple

from fastapi_cache.types import Backend

Expand All @@ -12,11 +19,104 @@ class Value:
ttl_ts: Optional[int]


class InMemoryBackend(Backend):
class _ExpirationIndex:
"""Exact (no stale entries) indexed min-heap of (expiry, key).

Unlike a lazy heap with generation tombstones, this needs at most
one heap entry per live cache key; overwrites and evictions remove
the old expiry in O(log n), without periodic O(n) heap compaction.
"""

def __init__(self) -> None:
self._heap: List[Tuple[int, str]] = []
self._positions: Dict[str, int] = {}

def __len__(self) -> int:
return len(self._heap)

def _swap(self, left: int, right: int) -> None:
items = self._heap
items[left], items[right] = items[right], items[left]
self._positions[items[left][1]] = left
self._positions[items[right][1]] = right

def _sift_up(self, index: int) -> None:
while index:
parent = (index - 1) // 2
if self._heap[parent] <= self._heap[index]:
return
self._swap(parent, index)
index = parent

def _sift_down(self, index: int) -> None:
length = len(self._heap)
while True:
left = index * 2 + 1
if left >= length:
return
right = left + 1
smallest = left
if right < length and self._heap[right] < self._heap[left]:
smallest = right
if self._heap[index] <= self._heap[smallest]:
return
self._swap(index, smallest)
index = smallest

def discard(self, key: str) -> None:
index = self._positions.pop(key, None)
if index is None:
return
last = self._heap.pop()
if index == len(self._heap):
return
self._heap[index] = last
self._positions[last[1]] = index
if index and self._heap[index] < self._heap[(index - 1) // 2]:
self._sift_up(index)
else:
self._sift_down(index)

def update(self, key: str, expires_at: Optional[int]) -> None:
self.discard(key)
if expires_at is None:
return
index = len(self._heap)
self._heap.append((expires_at, key))
self._positions[key] = index
self._sift_up(index)

def oldest_expired(self, now: int) -> Optional[str]:
if self._heap and self._heap[0][0] < now:
return self._heap[0][1]
return None


class InMemoryBackend(Backend):
def __init__(self, *, max_entries: Optional[int] = None) -> None:
if max_entries is not None and (
isinstance(max_entries, bool)
or not isinstance(max_entries, int)
or max_entries < 1
):
raise ValueError("max_entries must be a positive integer or None")
# Capacity is fixed at construction: runtime resizing would require
# coordinated eviction and index rebuilding under the async lock.
self._max_entries = max_entries
self._store: Dict[str, Value] = {}
self._recency: Optional[OrderedDict[str, None]] = (
OrderedDict() if max_entries is not None else None
)
self._expiry_index: Optional[_ExpirationIndex] = (
_ExpirationIndex() if max_entries is not None else None
)
self._lock: Optional[Lock] = None

@property
def max_entries(self) -> Optional[int]:
"""Immutable per-instance entry cap, or None for legacy unbounded mode."""
return self._max_entries

def _get_lock(self) -> Lock:
if self._lock is None:
self._lock = Lock()
Expand All @@ -26,12 +126,21 @@ def _get_lock(self) -> Lock:
def _now(self) -> int:
return int(time.time())

def _discard(self, key: str) -> None:
self._store.pop(key, None)
if self._recency is not None:
self._recency.pop(key, None)
if self._expiry_index is not None:
self._expiry_index.discard(key)

def _get(self, key: str) -> Optional[Value]:
v = self._store.get(key)
if v:
if v.ttl_ts is not None and v.ttl_ts < self._now:
del self._store[key]
self._discard(key)
else:
if self._recency is not None:
self._recency.move_to_end(key)
return v
return None

Expand All @@ -54,7 +163,23 @@ async def get(self, key: str) -> Optional[bytes]:
async def set(self, key: str, value: bytes, expire: Optional[int] = None) -> None:
async with self._get_lock():
ttl_ts = None if expire is None else self._now + expire
if self._recency is not None and self.max_entries is not None:
# Each new key needs at most ONE freed slot. Reclaim the
# earliest expired entry first, not an arbitrary live LRU
# entry. No scan and no unbounded cleanup under the lock.
if key not in self._store and len(self._store) >= self.max_entries:
assert self._expiry_index is not None # noqa: S101
expired = self._expiry_index.oldest_expired(self._now)
if expired is not None:
self._discard(expired)
else:
self._discard(next(iter(self._recency)))
self._store[key] = Value(value, ttl_ts)
if self._recency is not None:
self._recency[key] = None
self._recency.move_to_end(key)
if self._expiry_index is not None:
self._expiry_index.update(key, ttl_ts)

async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None) -> int:
count = 0
Expand All @@ -64,10 +189,10 @@ async def clear(self, namespace: Optional[str] = None, key: Optional[str] = None
keys = list(self._store.keys())
for cache_key in keys:
if cache_key.startswith(boundary):
del self._store[cache_key]
self._discard(cache_key)
count += 1
elif key:
if key in self._store:
del self._store[key]
self._discard(key)
count += 1
return count
102 changes: 102 additions & 0 deletions scripts/benchmark_bounded_inmemory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
"""Bounded InMemory scaling gate: fill, full-capacity LRU and TTL reclaim.

Run on both legacy Python 3.8 and supported modern Python. This test
benchmarks pure backend operations (not real request latency).
"""

import argparse
import asyncio
import json
import sys
import time
from typing import Dict, Tuple

from fastapi_cache.backends.inmemory import InMemoryBackend


class _FixedClockBackend(InMemoryBackend):
"""Deterministic TTL benchmark: no real-time sleep or machine clock drift."""

def __init__(self, capacity: int) -> None:
super().__init__(max_entries=capacity)
self.now = 1000

@property
def _now(self) -> int:
return self.now


async def measure(capacity: int) -> Tuple[Dict[str, float], Dict[str, int]]:
value = b"small-payload"
backend = _FixedClockBackend(capacity)
start = time.perf_counter()
for i in range(capacity):
await backend.set("initial:" + str(i), value, expire=600)
fill = time.perf_counter() - start

# Run a complete second capacity worth of NEW keys after the cache
# fills: every write MUST evict the least-recently-used live key.
start = time.perf_counter()
for i in range(capacity):
await backend.set("churn:" + str(i), value, expire=600)
churn = time.perf_counter() - start
if len(backend._store) != capacity: # pyright: ignore[reportPrivateUsage]
raise AssertionError("capacity violated during full LRU churn")

expired_backend = _FixedClockBackend(capacity)
for i in range(capacity):
await expired_backend.set("stale:" + str(i), value, expire=1)
expired_backend.now = 1002 # All old entries have expired.
start = time.perf_counter()
for i in range(capacity):
await expired_backend.set("reclaimed:" + str(i), value, expire=600)
expiry_reclaim = time.perf_counter() - start
if len(expired_backend._store) != capacity: # pyright: ignore[reportPrivateUsage]
raise AssertionError("capacity violated during TTL reclaim")
if await expired_backend.get("stale:0") is not None:
raise AssertionError("expired keys were not reclaimed")

return (
{"fill": fill, "full_lru_churn": churn, "full_expiry_reclaim": expiry_reclaim},
{"fill": capacity, "full_lru_churn": capacity, "full_expiry_reclaim": capacity},
)


def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--assert-scaling", action="store_true")
args = parser.parse_args()

asyncio.run(measure(300)) # Warm up imports/loop and code paths.
results = {}
for capacity in (10_000, 100_000):
durations, counts = asyncio.run(measure(capacity))
results[capacity] = {
"seconds": durations,
"ms_per_write": {
key: durations[key] * 1000 / counts[key] for key in durations
},
}
growth = {
key: results[100_000]["ms_per_write"][key] /
results[10_000]["ms_per_write"][key]
for key in ("fill", "full_lru_churn", "full_expiry_reclaim")
}
print(json.dumps({
"python": sys.version.split()[0],
"capacity": [10_000, 100_000],
"measurements": results,
"per_write_growth_100k_vs_10k": growth,
"note": "CI microbenchmark, not endpoint latency or payload-byte bound",
}, indent=2))
if args.assert_scaling:
for operation, ratio in growth.items():
if ratio > 4.5:
raise AssertionError(
f"{operation} grew {ratio:.2f}x per write from 10k to 100k; "
"an O(N) scan or regression is likely."
)


if __name__ == "__main__":
main()
Loading
Loading