fix(cmd): preserve bytes in streamed output and diagnostics

Strict text decoding could make fetch/push output readers fail after Git
successfully transferred refs, leaving empty result lists. Clone diagnostics,
progress messages, failed-process stderr and timeout messages could likewise
raise Unicode errors when names contained arbitrary bytes.

Use the shared lossless helpers for output pumps, progress parsing and error
re-encoding. Default subprocess text streams to surrogateescape while respecting
an explicit errors policy. Keep binary streams binary and retain existing
newline, timeout and process-finalization behavior.

Nine regressions demonstrate the former failures and pass after the fix. They
cover synchronous/streamed output, binary pumps, saved stderr, timeout and
clone diagnostics, and actual SHA-1/SHA-256 fetch/push results. Reftable fixtures
carry byte refs without invalid loose filenames or byte-valued refspec arguments.
The connected Gix command/remote run passes 295 tests and four subtests; its
remaining remote test passes with the project runner's isolated Git config
instead of inherited fetch.prune=true. Ruff and mypy pass.

Author: codex

doc/source/changes.rst

@@ -47,6 +47,9 @@ Git executable and safety
 * Known command outcomes retain established GitPython/configparser exceptions
   where distinguishable. Other failures raise ``GitCommandError`` with Git's
   exit status and diagnostic output; exact error text may differ.
+  Text and streamed command output preserve undecodable bytes with
+  ``surrogateescape``, including fetch/push results, clone errors and timeout
+  diagnostics containing byte names.
 
 API changes
 -----------

git/cmd.py

@@ -43,7 +43,7 @@
     overload,
 )
 
-from git.compat import defenc, force_bytes, safe_decode
+from git.compat import defenc, safe_decode, safe_encode
 from git import _backend
 from git.exc import (
     CommandError,
@@ -199,7 +199,7 @@ def pump_stream(
                 if handler:
                     if is_decode:
                         assert isinstance(line, bytes)
-                        line_str = line.decode(defenc)
+                        line_str = safe_decode(line)
                         handler(line_str)
                     else:
                         handler(line)
@@ -451,15 +451,15 @@ def wait(self, stderr: Union[None, str, bytes] = b"") -> int:
             status = self.status
             p_stderr = None
 
-        stderr_b = force_bytes(data=stderr, encoding="utf-8") or b""
+        stderr_b = safe_encode(stderr) or b""
         if self._timeout_error is not None:
             stderr_b += (b"\n" if stderr_b else b"") + self._timeout_error.encode("utf-8")
             status = status or 1
 
         def read_all_from_possibly_closed_stream(stream: Union[IO[bytes], None]) -> bytes:
             if stream:
                 try:
-                    return stderr_b + force_bytes(stream.read())
+                    return stderr_b + safe_encode(stream.read())
                 except (OSError, ValueError):
                     return stderr_b or b""
             else:
@@ -1457,8 +1457,9 @@ def execute(
             If ``True``, default ``True``, we open stdout on the created process.
 
         :param universal_newlines:
-            If ``True``, pipes will be opened as text, and lines are split at all known
-            line endings.
+            If ``True``, pipes are opened as text with CR, LF and CRLF newline
+            handling. Undecodable bytes are preserved with ``surrogateescape``
+            unless an explicit ``errors`` subprocess argument selects another policy.
 
         :param shell:
             Whether to invoke commands through a shell
@@ -1569,6 +1570,8 @@ def execute(
             shell,
             universal_newlines,
         )
+        if universal_newlines:
+            subprocess_kwargs.setdefault("errors", "surrogateescape")
         try:
             proc = safer_popen(
                 command,
@@ -1609,7 +1612,7 @@ def kill_process(pid: int) -> None:
 
             def make_timeout_error() -> Union[str, bytes]:
                 err = f'Timeout: the command "{" ".join(redacted_command)}" did not complete in {timeout:g} secs.'
-                return err if universal_newlines else err.encode(defenc)
+                return err if universal_newlines else safe_encode(err)
 
             def communicate() -> Tuple[Union[str, bytes, None], Union[str, bytes, None]]:
                 assert watchdog is not None

git/util.py

@@ -85,6 +85,7 @@
     from git.remote import Remote
     from git.repo.base import Repo
 
+from git.compat import safe_decode
 from git.types import (
     HSH_TD,
     Files_TD,
@@ -728,7 +729,7 @@ def _parse_progress_line(self, line: AnyStr) -> Optional[object]:
         # Compressing objects: 100% (2/2)
         # Compressing objects: 100% (2/2), done.
         if isinstance(line, bytes):  # mypy argues about ternary assignment.
-            line_str = line.decode("utf-8")
+            line_str = safe_decode(line)
         else:
             line_str = line
         self._cur_line = line_str

test/test_byte_output.py

@@ -0,0 +1,90 @@
+"""Git output carries paths and reference names with arbitrary bytes."""
+
+import os
+import sys
+
+import pytest
+
+from git import Git, GitCommandError, Repo, RemoteProgress
+from git.cmd import handle_process_output
+from git.compat import safe_decode, safe_encode
+from test.lib.fixtures import writable_fixture
+
+
+@pytest.mark.parametrize("as_process", [False, True])
+def test_text_process_output_preserves_bytes(as_process):
+    proc = Git().execute(
+        [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'name-\\xff\\n')"],
+        universal_newlines=True,
+        as_process=as_process,
+    )
+    if as_process:
+        lines = []
+        handle_process_output(proc, lines.append, None, decode_streams=False)
+        proc.wait()
+        assert safe_encode("".join(lines)) == b"name-\xff\n"
+    else:
+        assert safe_encode(proc) == b"name-\xff"
+
+
+def test_binary_process_output_handler_preserves_bytes():
+    proc = Git().execute(
+        [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'name-\\xff\\n')"], as_process=True
+    )
+    lines = []
+    handle_process_output(proc, lines.append, None)
+    proc.wait()
+    assert safe_encode("".join(lines)) == b"name-\xff\n"
+
+
+def test_process_error_preserves_previously_read_bytes():
+    proc = Git().execute([sys.executable, "-c", "raise SystemExit(1)"], as_process=True)
+    with pytest.raises(GitCommandError) as error:
+        proc.wait(stderr=safe_decode(b"fatal: name-\xff"))
+    assert "name-\udcff" in str(error.value)
+
+
+def test_progress_preserves_byte_diagnostics():
+    progress = RemoteProgress()
+    progress._parse_progress_line(b"fatal: name-\xff")
+    assert safe_encode(progress.error_lines[0]) == b"fatal: name-\xff"
+
+
+@pytest.mark.skipif(os.name == "nt", reason="POSIX timeout and byte-argument handling")
+def test_timeout_preserves_byte_arguments():
+    with pytest.raises(GitCommandError) as error:
+        Git().execute(
+            [sys.executable, "-c", "import time; time.sleep(5)", safe_decode(b"name-\xff")],
+            kill_after_timeout=0.1,
+        )
+    assert "name-\udcff" in str(error.value)
+
+
+@pytest.mark.skipif(os.name == "nt", reason="Windows arguments cannot express arbitrary path bytes")
+def test_clone_preserves_byte_path_in_failure():
+    with writable_fixture("remote-byte-output.sh", "sha1") as temporary:
+        root = temporary.path()
+        with pytest.raises(GitCommandError) as error:
+            Repo.clone_from(root / safe_decode(b"missing-\xff"), root / "clone")
+        assert "missing-\udcff" in str(error.value)
+
+
+@pytest.mark.parametrize("object_format", ["sha1", "sha256"])
+def test_fetch_and_push_preserve_byte_reference_results(object_format):
+    with writable_fixture("remote-byte-output.sh", object_format) as temporary:
+        root = temporary.path()
+        with Repo(root / "repo") as repo:
+            origin = repo.create_remote("origin", str(root / "source"))
+            fetched = origin.fetch()
+            assert {safe_encode(info.ref.path) for info in fetched} == {
+                b"refs/remotes/origin/main",
+                b"refs/remotes/origin/branch-\xff",
+            }
+        with Repo(root / "source") as repo:
+            target = repo.create_remote("target", str(root / "target"))
+            pushed = target.push("refs/heads/*:refs/heads/*")
+            assert not pushed.error
+            assert {safe_encode(info.remote_ref_string) for info in pushed} == {
+                b"refs/heads/main",
+                b"refs/heads/branch-\xff",
+            }

tests/fixtures/remote-byte-output.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -eu
+git init --initial-branch=main --object-format="$1" --ref-format=reftable source
+git -C source commit --allow-empty -m initial
+oid=$(git -C source rev-parse HEAD)
+printf 'create refs/heads/branch-\377\0%s\0' "$oid" | git -C source update-ref --stdin -z
+git init --initial-branch=main --object-format="$1" --ref-format=reftable repo
+git init --bare --object-format="$1" --ref-format=reftable target