diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index c45450b398a5d8b..1f89c12f223c7d9 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -45,34 +45,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - _PyTok_Loc loc; - _PyTok_Line line; - if (check(_PyTok_SourceLocation( - &source, 0, _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate empty source") < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong empty source location") < 0 || - check(_PyTok_SourceLine(&source, 1, &line) == 0, - "cannot find empty source line") < 0 || - check(line.start == 0 && line.end == 0, - "wrong empty source line") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, - "accepted empty source line") < 0 || + if (check_system_error( + _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + "accepted empty source line") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, "accepted multiple source lines") < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0) { - goto error; - } - - if (check(_PyTok_SourceAppendLine(&source, "alpha\n", 6, 0) == 0, - "wrong first source offset") < 0 || + "accepted missing implicit newline") < 0 || + check(_PyTok_SourceAppendLine( + &source, "alpha\n", 6, 0) == 0, + "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0) { + "wrong second source offset") < 0 || + check(!_PyTok_SourceLineIsImplicit(&source, 1) && + _PyTok_SourceLineIsImplicit(&source, 2), + "wrong implicit newline flags") < 0) { goto error; } @@ -84,159 +74,24 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(_PyTok_SourceAppendLine(&source, "nul\0x\n", 6, 0) == 9, - "wrong third source offset") < 0) { - goto error; - } - - int marker_line = 257; - int final_line = 300; - _PyTok_Off marker_start = -1; - for (int lineno = 4; lineno <= final_line; lineno++) { - const char *text = lineno == marker_line ? "marker\n" : "x\n"; - Py_ssize_t len = (Py_ssize_t)strlen(text); - _PyTok_Off start = _PyTok_SourceAppendLine( - &source, text, len, lineno == final_line); - if (start < 0) { - goto error; - } - if (lineno == marker_line) { - marker_start = start; - } - } - - if (check(source.nlines == final_line, "wrong source line count") < 0 || - check(_PyTok_SourceLine(&source, marker_line, &line) == 0, - "cannot find late source line") < 0 || - check(line.start == marker_start && - line.end == marker_start + 7, - "wrong late source line") < 0 || - check(!line.implicit_newline && !line.contains_nul, - "wrong late source flags") < 0 || - check(_PyTok_SourceLine(&source, 2, &line) == 0, - "cannot find second source line") < 0 || - check(line.start == 6 && line.end == 9 && - line.implicit_newline && !line.contains_nul, - "wrong second source line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong early implicit newline flags") < 0 || - check(_PyTok_SourceLine(&source, 3, &line) == 0, - "cannot find third source line") < 0 || - check(line.contains_nul, "missing null byte flag") < 0 || - check(_PyTok_SourceLine(&source, final_line, &line) == 0, - "cannot find final source line") < 0 || - check(line.implicit_newline && - _PyTok_SourceLineIsImplicit(&source, final_line), - "missing late implicit newline flag") < 0) { - goto error; - } - - Py_ssize_t view_len; - const char *view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(6, 8), &view_len); - if (check(view != NULL && view_len == 2 && - memcmp(view, "\xce\xb2", 2) == 0, - "wrong source span view") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left line boundary") < 0 || - check(loc.lineno == marker_line - 1 && loc.byte_col == 2, - "wrong left boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right line boundary") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 0, - "wrong right boundary location") < 0 || - check(_PyTok_SourceLocation( - &source, marker_start + 1, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate late source byte") < 0 || - check(loc.lineno == marker_line && loc.byte_col == 1, - "wrong late source location") < 0) { - goto error; - } - - if (check(_PyTok_SourceLocation( - &source, source.len, _PYTOK_AFFINITY_LEFT, &loc) == 0, - "cannot locate left EOF") < 0 || - check(loc.lineno == final_line && loc.byte_col == 2, - "wrong left EOF location") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate right EOF") < 0 || - check(loc.lineno == final_line + 1 && loc.byte_col == 0, - "wrong right EOF location") < 0 || - check(_PyTok_SourceLine(&source, final_line + 1, &line) == 0, - "cannot find virtual EOF line") < 0 || - check(line.start == source.len && line.end == source.len, - "wrong virtual EOF line") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 0) && - !_PyTok_SourceLineIsImplicit( - &source, final_line + 1), - "virtual or invalid line is implicit") < 0) { - goto error; - } - - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, source.len + 1), &view_len); - if (check_system_error(view == NULL, "accepted invalid source span") < 0 || - check_system_error( - _PyTok_SourceLocation( - &source, source.len + 1, - _PYTOK_AFFINITY_RIGHT, &loc) < 0, - "accepted invalid source offset") < 0 || - check_system_error( - _PyTok_SourceLine(&source, final_line + 2, &line) < 0, - "accepted invalid source line") < 0) { + if (check(source.len == 9 && + memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - _PyTok_SourceInit(&source); if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || check_system_error( _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, - "appended after unterminated source line") < 0 || - check(_PyTok_SourceLocation( - &source, source.len, - _PYTOK_AFFINITY_RIGHT, &loc) == 0, - "cannot locate unterminated EOF") < 0 || - check(loc.lineno == 1 && loc.byte_col == 4, - "wrong unterminated EOF location") < 0) { + "appended after unterminated source line") < 0) { goto error; } - if (check_line_view(&source, 1, "tail") < 0 || check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) { goto error; } - _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, - "wrong retained source offset") < 0 || - _PyTok_SourceLine(&source, 1, &line) < 0 || - check(line.start == 4 && line.end == 6, - "wrong retained source line") < 0 || - _PyTok_SourceLocation( - &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || - check(loc.lineno == 1 && loc.byte_col == 0, - "wrong retained source location") < 0) { - goto error; - } - view = _PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(4, 5), &view_len); - if (check(view != NULL && view_len == 1 && view[0] == 'a', - "wrong retained source span") < 0 || - check_system_error(_PyTok_SourceSpanView( - &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, - "accepted discarded source span") < 0) { - goto error; - } - _PyTok_SourceClear(&source); Py_RETURN_NONE; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 989944afd9fea96..d69eab93923e81c 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -2,8 +2,6 @@ #include "source.h" -#define LINE_CHECKPOINT_INTERVAL 256 - void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -14,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->line_checkpoints); PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -74,34 +71,6 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_checkpoints(_PyTok_SourceText *source, int needed) -{ - if (needed <= source->checkpoints_cap) { - return 0; - } - int cap; - if (source->checkpoints_cap == 0) { - cap = 16; - } - else if (source->checkpoints_cap <= INT_MAX / 2) { - cap = source->checkpoints_cap * 2; - } - else { - PyErr_NoMemory(); - return -1; - } - _PyTok_Off *checkpoints = source->line_checkpoints; - PyMem_Resize(checkpoints, _PyTok_Off, cap); - if (checkpoints == NULL) { - PyErr_NoMemory(); - return -1; - } - source->line_checkpoints = checkpoints; - source->checkpoints_cap = cap; - return 0; -} - static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { @@ -165,11 +134,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, return -1; } int nlines = source->nlines + 1; - int checkpoint = ((nlines - 1) % LINE_CHECKPOINT_INTERVAL) == 0; - int checkpoint_count = (nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - if ((checkpoint && - reserve_checkpoints(source, checkpoint_count) < 0) || - (implicit_newline && reserve_implicit_lines(source, nlines) < 0) || + if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -178,10 +143,6 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = - source->base_offset + start; - } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); @@ -210,19 +171,6 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, return line; } -const char * -_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, - Py_ssize_t *len) -{ - if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || - span.end - source->base_offset > source->len || len == NULL) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); - return NULL; - } - *len = span.end - span.start; - return _PyTok_SourceData(source) + (span.start - source->base_offset); -} - int _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) { @@ -233,124 +181,3 @@ _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) return (source->implicit_lines[(lineno - 1) / 8] >> ((lineno - 1) & 7)) & 1; } - -static int -source_ends_in_newline(const _PyTok_SourceText *source) -{ - return source->len > 0 && source->bytes[source->len - 1] == '\n'; -} - -static int -eof_lineno(const _PyTok_SourceText *source) -{ - if (source->nlines == 0) { - return 1; - } - return source->nlines + source_ends_in_newline(source); -} - -int -_PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, - _PyTok_Line *line) -{ - if (line == NULL || lineno < 1 || lineno > eof_lineno(source)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); - return -1; - } - if (lineno > source->nlines) { - *line = (_PyTok_Line){ - .start = source->base_offset + source->len, - .end = source->base_offset + source->len, - }; - return 0; - } - - int checkpoint = (lineno - 1) / LINE_CHECKPOINT_INTERVAL; - int current = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (current < lineno) { - start = _PyTok_SourceFindLineEnd(source, start); - if (start < 0) { - return -1; - } - current++; - } - _PyTok_Off end = source->base_offset + source->len; - if (lineno < source->nlines) { - end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - } - *line = (_PyTok_Line){ - .start = start, - .end = end, - .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), - .contains_nul = memchr( - source->bytes + (start - source->base_offset), - 0, end - start) != NULL, - }; - return 0; -} - -int -_PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, - _PyTok_Affinity affinity, _PyTok_Loc *loc) -{ - if (offset < source->base_offset || - offset - source->base_offset > source->len || loc == NULL || - (affinity != _PYTOK_AFFINITY_LEFT && - affinity != _PYTOK_AFFINITY_RIGHT)) { - PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); - return -1; - } - if (source->nlines == 0 || - (offset - source->base_offset == source->len && - source_ends_in_newline(source) && - affinity == _PYTOK_AFFINITY_RIGHT)) { - *loc = (_PyTok_Loc){eof_lineno(source), 0}; - return 0; - } - - _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { - key--; - } - int low = 0; - int high = (source->nlines - 1) / LINE_CHECKPOINT_INTERVAL + 1; - while (low < high) { - int middle = low + (high - low) / 2; - if (source->line_checkpoints[middle] <= key) { - low = middle + 1; - } - else { - high = middle; - } - } - int checkpoint = low - 1; - if (checkpoint < 0) { - PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); - return -1; - } - int lineno = checkpoint * LINE_CHECKPOINT_INTERVAL + 1; - _PyTok_Off start = source->line_checkpoints[checkpoint]; - while (lineno < source->nlines) { - _PyTok_Off end = _PyTok_SourceFindLineEnd(source, start); - if (end < 0) { - return -1; - } - if (offset < end || - (offset == end && affinity == _PYTOK_AFFINITY_LEFT)) { - break; - } - start = end; - lineno++; - } - _PyTok_Off byte_col = offset - start; - if (byte_col > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "tokenizer column is too large"); - return -1; - } - *loc = (_PyTok_Loc){lineno, (int)byte_col}; - return 0; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 2d9c4f479123096..6972caf8a36aa24 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -18,28 +18,13 @@ typedef struct { int byte_col; } _PyTok_Loc; -typedef enum { - _PYTOK_AFFINITY_LEFT, - _PYTOK_AFFINITY_RIGHT, -} _PyTok_Affinity; - -/* The half-open range includes the terminating newline when present. */ -typedef struct { - _PyTok_Off start; - _PyTok_Off end; - unsigned implicit_newline : 1; - unsigned contains_nul : 1; -} _PyTok_Line; - typedef struct { char *bytes; _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - _PyTok_Off *line_checkpoints; unsigned char *implicit_lines; int nlines; - int checkpoints_cap; Py_ssize_t implicit_cap; } _PyTok_SourceText; @@ -68,20 +53,9 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* The returned view is invalidated by SourceAppendLine and SourceClear. */ -PyAPI_FUNC(const char *) _PyTok_SourceSpanView( - const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line in the retained window. Empty and newline-terminated - sources have an empty virtual line at EOF. */ -PyAPI_FUNC(int) _PyTok_SourceLine( - const _PyTok_SourceText *, int, _PyTok_Line *); /* Return false for invalid line numbers and the virtual EOF line. */ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( const _PyTok_SourceText *, int); -/* At a line boundary, left affinity selects the preceding line at its end; - right affinity selects the following line at byte column zero. */ -PyAPI_FUNC(int) _PyTok_SourceLocation( - const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *); static inline _PyTok_Span _PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) @@ -95,24 +69,4 @@ _PyTok_SpanIsValid(_PyTok_Span span) return span.start >= 0 && span.end >= span.start; } -static inline _PyTok_Off -_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) -{ - if (source->bytes == NULL || start < source->base_offset || - start - source->base_offset >= source->len) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - _PyTok_Off relative_start = start - source->base_offset; - const char *newline = memchr( - source->bytes + relative_start, '\n', source->len - relative_start); - if (newline == NULL) { - PyErr_SetString(PyExc_SystemError, - "corrupt tokenizer source line index"); - return -1; - } - return source->base_offset + (newline - source->bytes) + 1; -} - #endif