Use magic for diff classification

diff/diff.go

@@ -10,13 +10,10 @@ import (
 	"strings"
 
 	"github.com/git-pkgs/archives"
+	"github.com/git-pkgs/magic"
 )
 
 const (
-	// binaryCheckSize is the number of bytes to check for null bytes when
-	// determining if content is binary.
-	binaryCheckSize = 8192
-
 	// Diff type constants for FileDiff.Type.
 	TypeModified = "modified"
 	TypeAdded    = "added"
@@ -133,7 +130,7 @@ func compareFile(path string, oldInfo, newInfo archives.FileInfo, oldReader, new
 	case !inOld && inNew:
 		fd := FileDiff{Path: path, Type: TypeAdded}
 		if content, err := readFileContent(newReader, path); err == nil {
-			if isBinary(content) {
+			if !isDiffableText(content) {
 				fd.IsBinary = true
 			} else {
 				fd.Diff = generateAddedDiff(path, content)
@@ -153,7 +150,7 @@ func compareFile(path string, oldInfo, newInfo archives.FileInfo, oldReader, new
 		}
 
 		fd := FileDiff{Path: path, Type: TypeModified}
-		if isBinary(oldContent) || isBinary(newContent) {
+		if !isDiffableText(oldContent) || !isDiffableText(newContent) {
 			fd.IsBinary = true
 		} else {
 			diffText, added, deleted := generateUnifiedDiff(path, oldContent, newContent)
@@ -176,25 +173,10 @@ func readFileContent(reader archives.Reader, path string) ([]byte, error) {
 	return io.ReadAll(rc)
 }
 
-// isBinary checks if content appears to be binary.
-func isBinary(content []byte) bool {
-	if len(content) == 0 {
-		return false
-	}
-
-	// Check first 8KB for null bytes
-	checkLen := len(content)
-	if checkLen > binaryCheckSize {
-		checkLen = binaryCheckSize
-	}
-
-	for i := 0; i < checkLen; i++ {
-		if content[i] == 0 {
-			return true
-		}
-	}
-
-	return false
+func isDiffableText(content []byte) bool {
+	detection := magic.Detect(content)
+	return detection.Kind == magic.KindText &&
+		(detection.Encoding == "" || detection.Encoding == "utf-8")
 }
 
 // generateUnifiedDiff generates a unified diff between two file contents.

diff/diff_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/git-pkgs/archives"
 )
 
+const previousBinaryCheckSize = 8 << 10
+
 func createTestArchiveWithFiles(files map[string]string) []byte {
 	buf := new(bytes.Buffer)
 	gw := gzip.NewWriter(buf)
@@ -101,24 +103,35 @@ func TestCompare(t *testing.T) {
 	}
 }
 
-func TestIsBinary(t *testing.T) {
+func TestIsDiffableText(t *testing.T) {
+	lateNUL := []byte(strings.Repeat("x", previousBinaryCheckSize))
+	lateNUL = append(lateNUL, 0)
+
 	tests := []struct {
 		name     string
 		content  []byte
-		expected bool
+		diffable bool
 	}{
-		{"empty", []byte{}, false},
-		{"text", []byte("hello world"), false},
-		{"binary with null", []byte{0x00, 0x01, 0x02}, true},
-		{"text with newlines", []byte("line1\nline2\nline3"), false},
-		{"json", []byte(`{"key": "value"}`), false},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := isBinary(tt.content)
-			if got != tt.expected {
-				t.Errorf("isBinary() = %v, want %v", got, tt.expected)
+		{"empty", nil, true},
+		{"text", []byte("hello world"), true},
+		{"Unicode text", []byte("café ☕"), true},
+		{"UTF-8 BOM", []byte("\xef\xbb\xbfhello"), true},
+		{"text with newlines", []byte("line1\nline2\nline3"), true},
+		{"JSON", []byte(`{"key": "value"}`), true},
+		{"PDF without NUL", []byte("%PDF-1.7\n"), false},
+		{"disallowed control", []byte("hello\x01world"), false},
+		{"invalid UTF-8", []byte{'c', 'a', 'f', 0xe9}, false},
+		{"early NUL", []byte("hello\x00world"), false},
+		{"late NUL", lateNUL, false},
+		{"UTF-16LE", []byte{0xff, 0xfe, 'h', 0, 'i', 0}, false},
+		{"UTF-16BE", []byte{0xfe, 0xff, 0, 'h', 0, 'i'}, false},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := isDiffableText(test.content)
+			if got != test.diffable {
+				t.Errorf("isDiffableText() = %v, want %v", got, test.diffable)
 			}
 		})
 	}
@@ -222,11 +235,11 @@ func TestCompareIdentical(t *testing.T) {
 
 func TestCompareBinaryFiles(t *testing.T) {
 	oldFiles := map[string]string{
-		"image.png": string([]byte{0x89, 0x50, 0x4E, 0x47, 0x00}), // Binary content
+		"document.pdf": "%PDF-1.7\nold content",
 	}
 
 	newFiles := map[string]string{
-		"image.png": string([]byte{0x89, 0x50, 0x4E, 0x47, 0x01}), // Different binary
+		"document.pdf": "%PDF-1.7\nnew content",
 	}
 
 	oldArchive, _ := archives.Open("old.tar.gz", bytes.NewReader(createTestArchiveWithFiles(oldFiles)))

go.mod

@@ -2,4 +2,7 @@ module github.com/git-pkgs/archives
 
 go 1.25.6
 
-require github.com/ulikunitz/xz v0.5.16
+require (
+	github.com/git-pkgs/magic v0.1.0
+	github.com/ulikunitz/xz v0.5.16
+)

go.sum

@@ -1,2 +1,4 @@
+github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY=
+github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI=
 github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
 github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=