gh-153568: Simplify parser identifier and buffer handling

Author: pablogsal

Parser/pegen.c

@@ -628,35 +628,11 @@ _PyPegen_new_identifier(Parser *p, const char *n)
     return NULL;
 }
 
-static expr_ty
-name_from_identifier(Parser *p, Token *t, PyObject *id)
-{
-    expr_ty result = _PyAST_Name(id, Load, t->lineno, t->col_offset,
-                                t->end_lineno, t->end_col_offset, p->arena);
-    if (result != NULL && _PyPegen_insert_memo(p, p->mark - 1, NAME, result) < 0) {
-        p->error_indicator = 1;
-        return NULL;
-    }
-    return result;
-}
-
-static expr_ty
-_PyPegen_name_from_token(Parser *p, Token* t)
+// Return an arena-owned identifier; callers borrow the reference.
+static PyObject *
+get_cached_identifier(Parser *p, PyObject *bytes)
 {
-    if (t == NULL) {
-        return NULL;
-    }
-    // Reuse the AST node when backtracking revisits this token. Memo lookup
-    // starts before the token and restores the position after it on a hit.
-    // Token kinds can be memo keys: generated grammar rule IDs start at 1000.
-    int mark = p->mark - 1;
-    p->mark = mark;
-    expr_ty cached = NULL;
-    if (_PyPegen_is_memoized(p, NAME, &cached)) {
-        return cached;
-    }
-    p->mark = mark + 1;
-    const char *s = PyBytes_AsString(t->bytes);
+    const char *s = PyBytes_AsString(bytes);
     if (!s) {
         p->error_indicator = 1;
         return NULL;
@@ -666,8 +642,8 @@ _PyPegen_name_from_token(Parser *p, Token* t)
     // arena-owned token bytes and values are arena-owned interned strings,
     // so borrowed references are valid for the lifetime of the parse
     // (including the second error pass, which reuses parser and arena).
-    Py_ssize_t len = PyBytes_GET_SIZE(t->bytes);
-    Py_hash_t hash = PyObject_Hash(t->bytes);
+    Py_ssize_t len = PyBytes_GET_SIZE(bytes);
+    Py_hash_t hash = PyObject_Hash(bytes);
     if (hash == -1) {
         p->error_indicator = 1;
         return NULL;
@@ -678,7 +654,7 @@ _PyPegen_name_from_token(Parser *p, Token* t)
             IDENTIFIER_CACHE_SIZE, sizeof(*p->identifier_cache));
         if (p->identifier_cache == NULL) {
             p->error_indicator = 1;
-            return (expr_ty)PyErr_NoMemory();
+            return PyErr_NoMemory();
         }
     }
     IdentifierCacheEntry *free_slot = NULL;
@@ -693,7 +669,7 @@ _PyPegen_name_from_token(Parser *p, Token* t)
         if (entry->hash == hash && entry->len == len &&
             memcmp(entry->key, s, len) == 0)
         {
-            return name_from_identifier(p, t, entry->value);
+            return entry->value;
         }
     }
     PyObject *id = _PyPegen_new_identifier(p, s);
@@ -707,7 +683,36 @@ _PyPegen_name_from_token(Parser *p, Token* t)
         free_slot->hash = hash;
         free_slot->value = id;
     }
-    return name_from_identifier(p, t, id);
+    return id;
+}
+
+static expr_ty
+_PyPegen_name_from_token(Parser *p, Token* t)
+{
+    if (t == NULL) {
+        return NULL;
+    }
+    // Reuse the AST node when backtracking revisits this token. Memo lookup
+    // starts before the token and restores the position after it on a hit.
+    // Token kinds can be memo keys: generated grammar rule IDs start at 1000.
+    int mark = p->mark - 1;
+    p->mark = mark;
+    expr_ty cached = NULL;
+    if (_PyPegen_is_memoized(p, NAME, &cached)) {
+        return cached;
+    }
+    p->mark = mark + 1;
+    PyObject *id = get_cached_identifier(p, t->bytes);
+    if (id == NULL) {
+        return NULL;
+    }
+    expr_ty result = _PyAST_Name(id, Load, t->lineno, t->col_offset,
+                                t->end_lineno, t->end_col_offset, p->arena);
+    if (result != NULL && _PyPegen_insert_memo(p, mark, NAME, result) < 0) {
+        p->error_indicator = 1;
+        return NULL;
+    }
+    return result;
 }
 
 expr_ty

Parser/tokenizer/decoder.c

@@ -80,15 +80,15 @@ normalize_newlines_into(char *result, const char *data, Py_ssize_t len,
                         Py_ssize_t *out_len, int *implicit_newline)
 {
     Py_ssize_t write = 0;
-    if (memchr(data, '\r', len) == NULL) {
-        // No carriage returns: nothing to translate, copy verbatim.
+    if (preserve_crlf || memchr(data, '\r', len) == NULL) {
+        // No translation needed: copy verbatim.
         memcpy(result, data, len);
         write = len;
     }
     else {
         for (Py_ssize_t read = 0; read < len; read++) {
             char c = data[read];
-            if (!preserve_crlf && c == '\r') {
+            if (c == '\r') {
                 if (read + 1 < len && data[read + 1] == '\n') {
                     read++;
                 }
@@ -341,24 +341,9 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
             }
             // Reserve space for an optional final '\n' and the NUL terminator.
             Py_ssize_t needed = line_len + 2;
-            if (needed > capacity) {
-                // Grow geometrically to avoid reallocating for every longer line.
-                Py_ssize_t next_capacity = capacity > 0 ? capacity : 256;
-                while (next_capacity < needed) {
-                    if (next_capacity > PY_SSIZE_T_MAX / 2) {
-                        next_capacity = needed;
-                        break;
-                    }
-                    next_capacity *= 2;
-                }
-                char *resized = PyMem_Realloc(normalized, next_capacity);
-                if (resized == NULL) {
-                    PyErr_NoMemory();
-                    tok->done = E_NOMEM;
-                    goto error;
-                }
-                normalized = resized;
-                capacity = next_capacity;
+            if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) {
+                tok->done = E_NOMEM;
+                goto error;
             }
             normalize_newlines_into(normalized, line, line_len,
                                     preserve_crlf, add_newline,

Parser/tokenizer/reader.c

@@ -37,13 +37,16 @@ _PyTok_ReaderFree(struct tok_state *tok)
     tok->reader = NULL;
 }
 
-static int
-reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
+int
+_PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed,
+                     Py_ssize_t initial_capacity)
 {
+    assert(initial_capacity > 0);
     if (needed <= *capacity) {
         return 0;
     }
-    Py_ssize_t cap = *capacity > 0 ? *capacity : BUFSIZ;
+    // Grow geometrically to avoid reallocating for every longer line.
+    Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity;
     while (cap < needed) {
         if (cap > PY_SSIZE_T_MAX / 2) {
             cap = needed;
@@ -72,8 +75,8 @@ append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len)
         reader->decoded_len = remaining;
     }
     if (len < 0 || reader->decoded_len > PY_SSIZE_T_MAX - len - 1 ||
-            reserve_buffer(&reader->decoded, &reader->decoded_cap,
-                           reader->decoded_len + len + 1) < 0) {
+            _PyTok_ReserveBuffer(&reader->decoded, &reader->decoded_cap,
+                                reader->decoded_len + len + 1, BUFSIZ) < 0) {
         PyErr_NoMemory();
         return -1;
     }
@@ -160,8 +163,8 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk)
     Py_ssize_t len = 0;
     for (;;) {
         if (len > PY_SSIZE_T_MAX - BUFSIZ ||
-                reserve_buffer(&reader->file_buffer, &reader->file_buffer_cap,
-                               len + BUFSIZ) < 0) {
+                _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap,
+                                    len + BUFSIZ, BUFSIZ) < 0) {
             return _PYTOK_READ_ERROR;
         }
         int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX);

Parser/tokenizer/reader_internal.h

@@ -63,6 +63,8 @@ typedef struct _PyTok_Reader {
 
 struct tok_state;
 
+// On failure, the buffer and capacity are unchanged.
+int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t);
 char *_PyTok_CopyBytes(const char *, Py_ssize_t);
 int _PyTok_DecodeOnce(
     struct tok_state *, _PyTok_Chunk *, const char *, const char *);