From 9f449d94e03c7c7287dac9b1489f637e16e754ac Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 11 Jul 2026 16:53:21 +0100 Subject: [PATCH] gh-153568: Skip newline translation for source without carriage returns The tokenizer copied every input byte by byte to normalize newlines, but source without a carriage return needs no translation and can be copied verbatim. --- ...7-11-16-53-21.gh-issue-153568.newlines.rst | 2 ++ Parser/tokenizer/decoder.c | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst new file mode 100644 index 000000000000000..deaa82c96fd3ac4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-16-53-21.gh-issue-153568.newlines.rst @@ -0,0 +1,2 @@ +Speed up the tokenizer by copying source text verbatim when it contains no +carriage returns. diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 5b588572f1fb7e8..ab8ae393d03032d 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -87,15 +87,22 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, return NULL; } Py_ssize_t write = 0; - for (Py_ssize_t read = 0; read < len; read++) { - char c = data[read]; - if (!preserve_crlf && c == '\r') { - if (read + 1 < len && data[read + 1] == '\n') { - read++; + if (memchr(data, '\r', len) == NULL) { + // No carriage returns: nothing to translate, copy verbatim. + memcpy(result, data, len); + write = len; + } + else { + for (Py_ssize_t read = 0; read < len; read++) { + char c = data[read]; + if (!preserve_crlf && c == '\r') { + if (read + 1 < len && data[read + 1] == '\n') { + read++; + } + c = '\n'; } - c = '\n'; + result[write++] = c; } - result[write++] = c; } int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; if (implicit) {