From aa480045755e2e6f7ad40a2e9230505ffae83998 Mon Sep 17 00:00:00 2001 From: lipengyu Date: Thu, 3 Sep 2026 20:56:20 +0800 Subject: [PATCH 1/3] gh-156891: Fix shlex source inclusion at EOF Fix shlex source inclusion when the source filename is the final token of its parent stream. Newly pushed streams now start with a valid lexical state, and popping a stream restores the parent state. --- Lib/shlex.py | 8 ++-- Lib/test/test_shlex.py | 41 +++++++++++++++++++ ...-09-03-20-54-42.gh-issue-156891.fliROC.rst | 2 + 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-03-20-54-42.gh-issue-156891.fliROC.rst diff --git a/Lib/shlex.py b/Lib/shlex.py index c7ffc918d53961..932af647c8530f 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -77,10 +77,12 @@ def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): newstream = StringIO(newstream) - self.filestack.appendleft((self.infile, self.instream, self.lineno)) + self.filestack.appendleft((self.infile, self.instream, self.lineno, + self.state)) self.infile = newfile self.instream = newstream self.lineno = 1 + self.state = ' ' if self.debug: if newfile is not None: print('shlex: pushing to file %s' % (self.infile,)) @@ -90,11 +92,11 @@ def push_source(self, newstream, newfile=None): def pop_source(self): "Pop the input source stack." self.instream.close() - (self.infile, self.instream, self.lineno) = self.filestack.popleft() + (self.infile, self.instream, self.lineno, + self.state) = self.filestack.popleft() if self.debug: print('shlex: popping to %s, line %d' \ % (self.instream, self.lineno)) - self.state = ' ' def get_token(self): "Get a token from the input stream (or from stack if it's nonempty)" diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 4c0cd88bbcda14..51246c666f6c44 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -420,6 +420,24 @@ def testPushSourceStream(self): s.push_source(io.StringIO("hello")) self.assertListEqual(list(s), ["hello", "world"]) + def testPushSourceKeepsPushback(self): + s = shlex.shlex("parent") + stream = io.StringIO("child") + s.push_token("pushed") + s.push_source(stream) + self.assertListEqual(list(s), ["pushed", "child", "parent"]) + self.assertTrue(stream.closed) + + def testPushSourceAfterEOF(self): + for posix in (False, True): + with self.subTest(posix=posix): + s = shlex.shlex("parent", posix=posix) + self.assertEqual(list(s), ["parent"]) + stream = io.StringIO("child") + s.push_source(stream) + self.assertEqual(list(s), ["child"]) + self.assertTrue(stream.closed) + def testPushSourceStreamDebug(self): s = shlex.shlex("") stream = io.StringIO("hello") @@ -514,6 +532,29 @@ def testSourceInclusion(self): s.sourcehook = lambda f: (f, io.StringIO("included")) self.assertEqual(list(s), ["included", "remaining"]) + def testSourceInclusionAtEOF(self): + for posix in (False, True): + with self.subTest(posix=posix): + s = shlex.shlex("trigger filename", posix=posix) + s.source = "trigger" + stream = io.StringIO("included") + s.sourcehook = lambda f: (f, stream) + self.assertEqual(list(s), ["included"]) + self.assertTrue(stream.closed) + + def testNestedSourceInclusionAtEOF(self): + for posix in (False, True): + with self.subTest(posix=posix): + streams = { + "child": io.StringIO("trigger grandchild"), + "grandchild": io.StringIO("included"), + } + s = shlex.shlex("trigger child", posix=posix) + s.source = "trigger" + s.sourcehook = lambda f: (f, streams[f]) + self.assertEqual(list(s), ["included"]) + self.assertTrue(all(stream.closed for stream in streams.values())) + def testGetTokenPopsPushbackDebug(self): s = shlex.shlex("") s.push_token("hello") diff --git a/Misc/NEWS.d/next/Library/2026-09-03-20-54-42.gh-issue-156891.fliROC.rst b/Misc/NEWS.d/next/Library/2026-09-03-20-54-42.gh-issue-156891.fliROC.rst new file mode 100644 index 00000000000000..fc93ead4e918a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-03-20-54-42.gh-issue-156891.fliROC.rst @@ -0,0 +1,2 @@ +Fix :class:`shlex.shlex` source inclusion when the source filename is the +final token in a stream. From d40079c9db24cfbfbf29e83002e2c8384d40fe9e Mon Sep 17 00:00:00 2001 From: lipengyu Date: Tue, 8 Sep 2026 11:19:44 +0800 Subject: [PATCH 2/3] update Reset the lexical state when switching to a new input source so a source filename at the end of its parent stream does not cause the child stream to be skipped. Co-Authored-By: lipengyu --- Lib/shlex.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/shlex.py b/Lib/shlex.py index 932af647c8530f..cafd0ded016c5f 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -77,8 +77,7 @@ def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." if isinstance(newstream, str): newstream = StringIO(newstream) - self.filestack.appendleft((self.infile, self.instream, self.lineno, - self.state)) + self.filestack.appendleft((self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream self.lineno = 1 @@ -92,11 +91,11 @@ def push_source(self, newstream, newfile=None): def pop_source(self): "Pop the input source stack." self.instream.close() - (self.infile, self.instream, self.lineno, - self.state) = self.filestack.popleft() + (self.infile, self.instream, self.lineno) = self.filestack.popleft() if self.debug: print('shlex: popping to %s, line %d' \ % (self.instream, self.lineno)) + self.state = ' ' def get_token(self): "Get a token from the input stream (or from stack if it's nonempty)" From 13fafc530d16f949b63225b182f190e91dd24a7c Mon Sep 17 00:00:00 2001 From: lipengyu Date: Tue, 8 Sep 2026 12:18:53 +0800 Subject: [PATCH 3/3] gh-156891: Remove unrelated shlex pushback test Keep the regression coverage focused on resetting lexical state when a new source is pushed. Co-authored-by: lipengyu --- Lib/test/test_shlex.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 51246c666f6c44..b74d7545c30817 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -420,14 +420,6 @@ def testPushSourceStream(self): s.push_source(io.StringIO("hello")) self.assertListEqual(list(s), ["hello", "world"]) - def testPushSourceKeepsPushback(self): - s = shlex.shlex("parent") - stream = io.StringIO("child") - s.push_token("pushed") - s.push_source(stream) - self.assertListEqual(list(s), ["pushed", "child", "parent"]) - self.assertTrue(stream.closed) - def testPushSourceAfterEOF(self): for posix in (False, True): with self.subTest(posix=posix):