diff --git a/diff/diff.go b/diff/diff.go index 36a88e4..0791869 100644 --- a/diff/diff.go +++ b/diff/diff.go @@ -10,13 +10,10 @@ import ( "strings" "github.com/git-pkgs/archives" + "github.com/git-pkgs/magic" ) const ( - // binaryCheckSize is the number of bytes to check for null bytes when - // determining if content is binary. - binaryCheckSize = 8192 - // Diff type constants for FileDiff.Type. TypeModified = "modified" TypeAdded = "added" @@ -133,7 +130,7 @@ func compareFile(path string, oldInfo, newInfo archives.FileInfo, oldReader, new case !inOld && inNew: fd := FileDiff{Path: path, Type: TypeAdded} if content, err := readFileContent(newReader, path); err == nil { - if isBinary(content) { + if !isDiffableText(content) { fd.IsBinary = true } else { fd.Diff = generateAddedDiff(path, content) @@ -153,7 +150,7 @@ func compareFile(path string, oldInfo, newInfo archives.FileInfo, oldReader, new } fd := FileDiff{Path: path, Type: TypeModified} - if isBinary(oldContent) || isBinary(newContent) { + if !isDiffableText(oldContent) || !isDiffableText(newContent) { fd.IsBinary = true } else { diffText, added, deleted := generateUnifiedDiff(path, oldContent, newContent) @@ -176,25 +173,10 @@ func readFileContent(reader archives.Reader, path string) ([]byte, error) { return io.ReadAll(rc) } -// isBinary checks if content appears to be binary. -func isBinary(content []byte) bool { - if len(content) == 0 { - return false - } - - // Check first 8KB for null bytes - checkLen := len(content) - if checkLen > binaryCheckSize { - checkLen = binaryCheckSize - } - - for i := 0; i < checkLen; i++ { - if content[i] == 0 { - return true - } - } - - return false +func isDiffableText(content []byte) bool { + detection := magic.Detect(content) + return detection.Kind == magic.KindText && + (detection.Encoding == "" || detection.Encoding == "utf-8") } // generateUnifiedDiff generates a unified diff between two file contents. diff --git a/diff/diff_test.go b/diff/diff_test.go index 5c63051..3694e72 100644 --- a/diff/diff_test.go +++ b/diff/diff_test.go @@ -10,6 +10,8 @@ import ( "github.com/git-pkgs/archives" ) +const previousBinaryCheckSize = 8 << 10 + func createTestArchiveWithFiles(files map[string]string) []byte { buf := new(bytes.Buffer) gw := gzip.NewWriter(buf) @@ -101,24 +103,35 @@ func TestCompare(t *testing.T) { } } -func TestIsBinary(t *testing.T) { +func TestIsDiffableText(t *testing.T) { + lateNUL := []byte(strings.Repeat("x", previousBinaryCheckSize)) + lateNUL = append(lateNUL, 0) + tests := []struct { name string content []byte - expected bool + diffable bool }{ - {"empty", []byte{}, false}, - {"text", []byte("hello world"), false}, - {"binary with null", []byte{0x00, 0x01, 0x02}, true}, - {"text with newlines", []byte("line1\nline2\nline3"), false}, - {"json", []byte(`{"key": "value"}`), false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isBinary(tt.content) - if got != tt.expected { - t.Errorf("isBinary() = %v, want %v", got, tt.expected) + {"empty", nil, true}, + {"text", []byte("hello world"), true}, + {"Unicode text", []byte("café ☕"), true}, + {"UTF-8 BOM", []byte("\xef\xbb\xbfhello"), true}, + {"text with newlines", []byte("line1\nline2\nline3"), true}, + {"JSON", []byte(`{"key": "value"}`), true}, + {"PDF without NUL", []byte("%PDF-1.7\n"), false}, + {"disallowed control", []byte("hello\x01world"), false}, + {"invalid UTF-8", []byte{'c', 'a', 'f', 0xe9}, false}, + {"early NUL", []byte("hello\x00world"), false}, + {"late NUL", lateNUL, false}, + {"UTF-16LE", []byte{0xff, 0xfe, 'h', 0, 'i', 0}, false}, + {"UTF-16BE", []byte{0xfe, 0xff, 0, 'h', 0, 'i'}, false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := isDiffableText(test.content) + if got != test.diffable { + t.Errorf("isDiffableText() = %v, want %v", got, test.diffable) } }) } @@ -222,11 +235,11 @@ func TestCompareIdentical(t *testing.T) { func TestCompareBinaryFiles(t *testing.T) { oldFiles := map[string]string{ - "image.png": string([]byte{0x89, 0x50, 0x4E, 0x47, 0x00}), // Binary content + "document.pdf": "%PDF-1.7\nold content", } newFiles := map[string]string{ - "image.png": string([]byte{0x89, 0x50, 0x4E, 0x47, 0x01}), // Different binary + "document.pdf": "%PDF-1.7\nnew content", } oldArchive, _ := archives.Open("old.tar.gz", bytes.NewReader(createTestArchiveWithFiles(oldFiles))) diff --git a/go.mod b/go.mod index 0cd8a90..58b9fc8 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/git-pkgs/archives go 1.25.6 -require github.com/ulikunitz/xz v0.5.16 +require ( + github.com/git-pkgs/magic v0.1.0 + github.com/ulikunitz/xz v0.5.16 +) diff --git a/go.sum b/go.sum index ae34018..94caf89 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ +github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= +github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=