diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst new file mode 100644 index 00000000000000..5d778139d06b50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst @@ -0,0 +1 @@ +Speed up the parser by caching repeated identifiers during a parse. diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f995095..3c34c18056ab95 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -572,11 +572,41 @@ _PyPegen_name_from_token(Parser *p, Token* t) p->error_indicator = 1; return NULL; } + // Identifiers repeat constantly; a small span-keyed cache skips the + // UTF-8 decode + intern for repeated occurrences. Keys point into + // arena-owned token bytes and values are arena-owned interned strings, + // so borrowed references are valid for the lifetime of the parse + // (including the second error pass, which reuses parser and arena). + Py_ssize_t len = PyBytes_GET_SIZE(t->bytes); + uint32_t hash = 2166136261u; + for (Py_ssize_t i = 0; i < len; i++) { + hash = (hash ^ (unsigned char)s[i]) * 16777619u; + } + struct _identifier_cache_entry *free_slot = NULL; + size_t idx = hash & (_PYPEGEN_IDENT_CACHE_SIZE - 1); + for (int probe = 0; probe < _PYPEGEN_IDENT_CACHE_MAX_PROBES; probe++) { + struct _identifier_cache_entry *e = + &p->ident_cache[(idx + probe) & (_PYPEGEN_IDENT_CACHE_SIZE - 1)]; + if (e->key == NULL) { + free_slot = e; + break; + } + if (e->hash == hash && e->len == len && memcmp(e->key, s, len) == 0) { + return _PyAST_Name(e->value, Load, t->lineno, t->col_offset, + t->end_lineno, t->end_col_offset, p->arena); + } + } PyObject *id = _PyPegen_new_identifier(p, s); if (id == NULL) { p->error_indicator = 1; return NULL; } + if (free_slot != NULL) { + free_slot->key = s; + free_slot->len = len; + free_slot->hash = hash; + free_slot->value = id; + } return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, p->arena); } @@ -844,6 +874,15 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, p->flags = flags; p->feature_version = feature_version; p->known_err_token = NULL; + p->ident_cache = PyMem_Calloc(_PYPEGEN_IDENT_CACHE_SIZE, + sizeof(struct _identifier_cache_entry)); + if (p->ident_cache == NULL) { + growable_comment_array_deallocate(&p->type_ignore_comments); + PyMem_Free(p->tokens[0]); + PyMem_Free(p->tokens); + PyMem_Free(p); + return (Parser *) PyErr_NoMemory(); + } p->level = 0; p->call_invalid_rules = 0; p->last_stmt_location.lineno = 0; @@ -859,6 +898,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, void _PyPegen_Parser_Free(Parser *p) { + PyMem_Free(p->ident_cache); Py_XDECREF(p->normalize); for (int i = 0; i < p->size; i++) { PyMem_Free(p->tokens[i]); diff --git a/Parser/pegen.h b/Parser/pegen.h index cbf44ba474fa39..6cb5de08b2c6b1 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -91,8 +91,17 @@ typedef struct { int call_invalid_rules; int debug; location last_stmt_location; + struct _identifier_cache_entry { + const char *key; // borrowed; points into arena-owned token bytes + Py_ssize_t len; + uint32_t hash; + PyObject *value; // borrowed; arena-owned interned identifier + } *ident_cache; } Parser; +#define _PYPEGEN_IDENT_CACHE_SIZE 2048 // power of two +#define _PYPEGEN_IDENT_CACHE_MAX_PROBES 8 + typedef struct { cmpop_ty cmpop; expr_ty expr;