Fix terminal directory globstar matching

CHANGES_1.in.rst

@@ -20,6 +20,7 @@ New features:
 
 Bug fixes:
 
+- `Pull #164`_: Keep direct files included for directory-only terminal globstar patterns such as ``d/**/``.
 - Hyperscan backend no longer fails on multibyte UTF-8 characters.
 - `Pull #123`_: Ignore invalid gitignore bracket ranges for `GitIgnoreSpec`.
 - `Pull #128`_: Support POSIX character classes (e.g. `[[:alpha:]]`) in gitignore bracket expressions.
@@ -195,3 +196,5 @@ Improvements:
 .. _`google-re2`: https://pypi.org/project/google-re2/
 .. _`hyperscan`: https://pypi.org/project/hyperscan/
 .. _`typing-extensions`: https://pypi.org/project/typing-extensions/
+
+.. _`Pull #164`: https://github.com/cpburnz/python-pathspec/pull/164

pathspec/patterns/gitignore/spec.py

@@ -69,6 +69,8 @@ class GitIgnoreSpecPattern(_GitIgnoreBasePattern):
 	def __normalize_segments(
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		requires_dir_descendant: bool = False,
 	) -> tuple[Optional[list[str]], Optional[str]]:
 		"""
 		Normalize the pattern segments to make processing easier.
@@ -79,6 +81,10 @@ def __normalize_segments(
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments. This may be modified in place.
 
+		*requires_dir_descendant* (:class:`bool`) indicates whether a terminal
+		directory globstar must consume a descendant segment before the directory
+		marker.
+
 		Raises :exc:`ValueError` if the pattern normalizes to nothing.
 
 		Returns a :class:`tuple` containing either:
@@ -151,6 +157,7 @@ def __normalize_segments(
 			and pattern_segs[0] == '**'
 			and pattern_segs[1] == '*'
 			and pattern_segs[2] == '**'
+			and not requires_dir_descendant
 		):
 			# The pattern "*/" will be normalized to "**/*/**" which will match every
 			# file not in the root directory. Special case this pattern for
@@ -264,11 +271,21 @@ def pattern_to_regex(
 		# Check whether the pattern is specifically a directory pattern before
 		# normalization.
 		is_dir_pattern = not orig_segs[-1]
+		# A terminal directory globstar with a concrete prefix must consume a
+		# descendant segment before the directory marker. Keep this semantic bit
+		# separate from normalized segments, which collapse adjacent ``**``.
+		requires_dir_descendant = (
+			is_dir_pattern
+			and len(orig_segs) > 2
+			and orig_segs[-2] == '**'
+			and any(seg not in ('', '**') for seg in orig_segs[:-2])
+		)
 
 		# Normalize pattern to make processing easier.
 		try:
 			pattern_segs, override_regex = cls.__normalize_segments(
 				is_dir_pattern, orig_segs,
+				requires_dir_descendant=requires_dir_descendant,
 			)
 		except ValueError as e:
 			raise GitIgnorePatternError((
@@ -284,6 +301,7 @@ def pattern_to_regex(
 			try:
 				regex_parts = cls.__translate_segments(
 					seg_errors, is_dir_pattern, pattern_segs,
+					requires_dir_descendant=requires_dir_descendant,
 				)
 			except (_PosixClassError, _RangeNotationError) as e:
 				if errors == 'raise':
@@ -325,6 +343,8 @@ def __translate_segments(
 		errors: Literal['literal', 'raise'],
 		is_dir_pattern: bool,
 		pattern_segs: list[str],
+		*,
+		requires_dir_descendant: bool = False,
 	) -> list[str]:
 		"""
 		Translate the pattern segments to regular expressions.
@@ -342,6 +362,10 @@ def __translate_segments(
 		*pattern_segs* (:class:`list` of :class:`str`) contains the pattern
 		segments.
 
+		*requires_dir_descendant* (:class:`bool`) indicates whether a terminal
+		directory globstar must consume a descendant segment before the directory
+		marker.
+
 		Raises :exc:`._PosixClassError` when an invalid POXIS class is found and
 		*errors* is ``'literal'``.
 
@@ -375,6 +399,8 @@ def __translate_segments(
 					# A normalized pattern ending with double-asterisks ('**') will match
 					# nonempty trailing path segments, not the parent directory itself.
 					if is_dir_pattern:
+						if requires_dir_descendant:
+							out_parts.append('(?s:/.+)')
 						out_parts.append(_DIR_MARK_CG)
 					else:
 						out_parts.append('/[^/]')

tests/test_06_gitignore.py

@@ -1074,6 +1074,107 @@ def test_b(self):
 					"vendor/.cache/y.txt",
 				}, debug)
 
+	def test_c(self):
+		"""
+		Test that terminal directory globstars require a descendant directory.
+		"""
+		cases = [
+			(
+				"d/**/",
+				{"d/", "d/top.txt"},
+				{
+					"d/child/",
+					"d/child/file.txt",
+					"d/child/grandchild/deep.txt",
+					"d/\nchild/",
+					"d/\nchild/file.txt",
+				},
+			),
+			(
+				"**/**/",
+				{"top.txt"},
+				{"d/", "d/file.txt", "a/b/deep.txt"},
+			),
+			(
+				"/d/**/",
+				{"d/top.txt"},
+				{"d/child/", "d/child/file.txt"},
+			),
+			(
+				"**/d/**/",
+				{"d/top.txt", "x/d/top.txt"},
+				{"d/child/file.txt", "x/d/child/file.txt"},
+			),
+			(
+				"a/**/**/",
+				{"a/top.txt"},
+				{"a/child/", "a/child/file.txt", "a/b/top.txt"},
+			),
+			(
+				"a/**/x/**/",
+				{"a/x/top.txt"},
+				{"a/x/child/file.txt", "a/b/x/child/file.txt"},
+			),
+			(
+				"*/**/",
+				{"d/top.txt"},
+				{"d/child/file.txt", "a/b/top.txt", "x/d/child/file.txt"},
+			),
+			(
+				"**/*/**/",
+				{"d/top.txt", "child/file.txt"},
+				{"d/child/file.txt", "a/x/child/file.txt", "child/grand/deep.txt"},
+			),
+		]
+		for pattern, unignored, ignored in cases:
+			for sub_test in self.parameterize_from_lines([pattern]):
+				with sub_test() as spec:
+					with self.subTest(pattern=pattern):
+						for path in unignored:
+							with self.subTest(path=path):
+								self.assertFalse(spec.match_file(path))
+						for path in ignored:
+							with self.subTest(path=path):
+								self.assertTrue(spec.match_file(path))
+
+	def test_d(self):
+		"""
+		Document terminal-globstar negation behavior without changing resolution.
+		"""
+		for sub_test in self.parameterize_from_lines([
+			"d/**/",
+			"!d/child/",
+		]):
+			with sub_test() as spec:
+				# PathSpec leaves the bare pattern-root directory unmatched so a
+				# traversal can still reach its direct files.
+				self.assertFalse(spec.match_file("d/"))
+				self.assertFalse(spec.match_file("d/top.txt"))
+				self.assertFalse(spec.match_file("d/child/"))
+				self.assertFalse(spec.match_file("d/child/top.txt"))
+				self.assertTrue(spec.match_file("d/child/grand/"))
+				# Existing flat last-match resolution lets the negated directory
+				# pattern match this descendant too. Git's ancestor-sensitive result
+				# differs, but changing that separate behavior is outside this fix.
+				self.assertFalse(spec.match_file("d/child/grand/deep.txt"))
+
+		for sub_test in self.parameterize_from_lines(["!d/**/"]):
+			with sub_test() as spec:
+				# The terminal-descendant rule applies to negated patterns too.
+				for path in ("d/", "d/top.txt"):
+					result = spec.check_file(path)
+					self.assertIsNone(result.include)
+					self.assertIsNone(result.index)
+				for path in (
+					"d/child/",
+					"d/child/file.txt",
+					"d/child/grand/deep.txt",
+				):
+					with self.subTest(negated_pattern_path=path):
+						result = spec.check_file(path)
+						self.assertFalse(result.include)
+						self.assertEqual(result.index, 0)
+
 
 class GitIgnoreSpecIssue139Test(GitIgnoreSpecMixin, TestCase):
 	"""