fix(tree): order cached entries by filename bytes

`TreeModifier.set_done()` sorted decoded Unicode names. A name ending in an
undecodable byte could sort before a valid UTF-8 name even when Git stores
them in the opposite order, changing iteration order after cache editing.

Recover each name's original bytes in the existing sort key and keep its
directory terminator. This matches the byte comparison in Git's
`tree.c::base_name_compare()` without changing tree construction, which still
belongs to Git or Gix.

The regression compares cache iteration with `git ls-tree -z` for byte and
emoji names. It fails before the fix and passes on both backends. Connected
tree tests pass on CLI and Gix; Ruff, ShellCheck and whitespace checks pass.

Author: codex

doc/source/changes.rst

@@ -63,7 +63,8 @@ API changes
   contain unquoted filename bytes.
 * Tree serialization now delegates construction to Git or the supported Gix
   backend. ``Tree.cache`` and ``TreeModifier`` edit one tree's in-memory entries;
-  object IDs and parent trees do not change automatically. Prefer
+  ``set_done()`` orders entries by filename bytes, including undecodable names.
+  Object IDs and parent trees do not change automatically. Prefer
   ``IndexFile.write_tree()`` for changes to staged paths.
 * ``Commit.stats`` preserves Unicode line separators in filenames when
   ``core.quotePath=false``. Git's LF-delimited output remains intact for root,

git/objects/tree.py

@@ -10,7 +10,7 @@
 
 import git.diff as git_diff
 from git.util import IterableList, join_path, to_bin_sha
-from git.compat import defenc, safe_decode
+from git.compat import defenc, safe_decode, safe_encode
 from git.cmd import Git
 
 from . import util
@@ -91,7 +91,7 @@ def set_done(self) -> "TreeModifier":
         :return:
             self
         """
-        self._cache.sort(key=lambda x: (x[2] + "/") if x[1] == Tree.tree_id << 12 else x[2])
+        self._cache.sort(key=lambda x: safe_encode(x[2]) + (b"/" if x[1] == Tree.tree_id << 12 else b""))
         return self
 
     # } END interface

test/test_tree.py

@@ -12,6 +12,7 @@
 
 from gitdb import IStream
 
+from git.compat import safe_encode
 from git.objects import Blob, Tree
 from git.objects.tree import TreeModifier
 from test.lib import TestBase, with_rw_repo
@@ -55,6 +56,14 @@ def test_invalid_tree_does_not_retain_partial_cache(object_format, invalid_posit
                 list(tree)
 
 
+def test_tree_modifier_orders_non_utf8_names_as_git_does():
+    with fixture_repo("tree-byte-order.sh") as repo:
+        expected = (read_only_fixture("tree-byte-order.sh") / "order").read_bytes().split(b"\0")[:-1]
+        tree = repo.head.commit.tree
+        tree.cache.set_done()
+        assert [safe_encode(item.name) for item in tree] == expected
+
+
 @ddt.ddt
 class TestTree(TestBase):
     @with_rw_repo("HEAD")

tests/fixtures/tree-byte-order.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eu
+git init --initial-branch=main repo
+cd repo
+blob=$(printf 'content\n' | git hash-object -w --stdin)
+tree=$({
+    printf '100644 blob %s\tname-\377\0' "$blob"
+    printf '100644 blob %s\tname-\360\237\222\251\0' "$blob"
+} | git mktree -z)
+commit=$(printf 'initial\n' | git commit-tree "$tree")
+git update-ref refs/heads/main "$commit"
+git ls-tree --name-only -z HEAD >../order