From 97abec1a8cd50882b055f64b557535c82ed8d7f9 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 31 Jul 2026 13:55:34 +0100 Subject: [PATCH 1/2] Detect extensionless archive formats --- README.md | 5 ++ archives.go | 71 +++++++++++++++++++------- archives_test.go | 124 +++++++++++++++++++++++++++++++++++++++++++-- hash_bench_test.go | 12 +++++ hash_test.go | 2 +- 5 files changed, 191 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 21f9c52..f4611d5 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,11 @@ The hash is computed over the archive as it was passed to `Open`, not the decomp reader, _ := archives.OpenBytes("pkg.tgz", data) ``` +Filename mappings are used first. When a name has no supported extension, +`Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, and xz from the content. +Compressed content is opened as TAR and returns a parser error when it does not +contain a TAR archive. + ### Prefix stripping Some package formats wrap content in a directory (npm uses `package/`). `OpenWithPrefix` strips a path prefix from all entries: diff --git a/archives.go b/archives.go index f7cadc0..1748d59 100644 --- a/archives.go +++ b/archives.go @@ -15,6 +15,18 @@ import ( "path" "strings" "time" + + "github.com/git-pkgs/magic" +) + +const ( + formatZIP = "zip" + formatTAR = "tar" + formatTarGzip = "tar.gz" + formatTGZ = "tgz" + formatTarBzip2 = "tar.bz2" + formatTarXZ = "tar.xz" + formatGem = "gem" ) // FileInfo represents metadata about a file in an archive. @@ -52,20 +64,23 @@ type Reader interface { } // Open creates an archive reader for the given content. -// The filename is used to detect the archive format. +// The filename is used first to detect the archive format. If it has no +// supported extension, the content is checked for a supported physical format. // The content reader will be read entirely into memory. // //nolint:ireturn // factory function returning interface by design func Open(filename string, content io.Reader) (Reader, error) { format := detectFormat(filename) - if format == "" { - return nil, fmt.Errorf("unsupported archive format: %s", filename) - } - raw, err := io.ReadAll(content) if err != nil { return nil, fmt.Errorf("reading archive content: %w", err) } + if format == "" { + format = detectContentFormat(raw) + } + if format == "" { + return nil, fmt.Errorf("unsupported archive format: %s", filename) + } return openRaw(format, raw) } @@ -77,6 +92,9 @@ func Open(filename string, content io.Reader) (Reader, error) { //nolint:ireturn // factory function returning interface by design func OpenBytes(filename string, content []byte) (Reader, error) { format := detectFormat(filename) + if format == "" { + format = detectContentFormat(content) + } if format == "" { return nil, fmt.Errorf("unsupported archive format: %s", filename) } @@ -87,23 +105,40 @@ func OpenBytes(filename string, content []byte) (Reader, error) { //nolint:ireturn func openRaw(format string, raw []byte) (Reader, error) { switch format { - case "zip": + case formatZIP: return openZip(raw) - case "tar": + case formatTAR: return openTar(raw, "") - case "tar.gz", "tgz": + case formatTarGzip, formatTGZ: return openTar(raw, "gzip") - case "tar.bz2": + case formatTarBzip2: return openTar(raw, "bzip2") - case "tar.xz": + case formatTarXZ: return openTar(raw, "xz") - case "gem": + case formatGem: return openGem(raw) default: return nil, fmt.Errorf("unsupported format: %s", format) } } +func detectContentFormat(content []byte) string { + switch magic.Detect(content).Format { + case "zip": + return formatZIP + case "tar": + return formatTAR + case "gzip": + return formatTarGzip + case "bzip2": + return formatTarBzip2 + case "xz": + return formatTarXZ + default: + return "" + } +} + // OpenWithPrefix opens an archive and strips the given prefix from all paths. // This is useful for npm packages which wrap content in a "package/" directory. // @@ -143,26 +178,26 @@ func detectFormat(filename string) string { // Check for compound extensions first if strings.HasSuffix(filename, ".tar.gz") { - return "tar.gz" + return formatTarGzip } if strings.HasSuffix(filename, ".tar.bz2") { - return "tar.bz2" + return formatTarBzip2 } if strings.HasSuffix(filename, ".tar.xz") { - return "tar.xz" + return formatTarXZ } // Check simple extensions ext := path.Ext(filename) switch ext { case ".zip", ".jar", ".whl", ".nupkg", ".egg": - return "zip" + return formatZIP case ".tar": - return "tar" + return formatTAR case ".tgz": - return "tgz" + return formatTGZ case ".gem": - return "gem" + return formatGem default: return "" } diff --git a/archives_test.go b/archives_test.go index bbdf59e..8a26f8c 100644 --- a/archives_test.go +++ b/archives_test.go @@ -5,6 +5,7 @@ import ( "archive/zip" "bytes" "compress/gzip" + "encoding/base64" "errors" "fmt" "io" @@ -12,6 +13,8 @@ import ( "strings" "testing" "time" + + "github.com/ulikunitz/xz" ) func TestDetectFormat(t *testing.T) { @@ -183,11 +186,9 @@ func TestZipReader(t *testing.T) { } } -// createTestTarGz creates a tar.gz archive in memory with test files -func createTestTarGz() []byte { +func createTestTar() []byte { buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - tw := tar.NewWriter(gw) + tw := tar.NewWriter(buf) files := []struct { name string @@ -210,10 +211,45 @@ func createTestTarGz() []byte { } _ = tw.Close() + return buf.Bytes() +} + +// createTestTarGz creates a tar.gz archive in memory with test files +func createTestTarGz() []byte { + buf := new(bytes.Buffer) + gw := gzip.NewWriter(buf) + _, _ = gw.Write(createTestTar()) _ = gw.Close() return buf.Bytes() } +func createTestTarBz2(t *testing.T) []byte { + t.Helper() + // The standard library provides a bzip2 reader but no writer. + const encoded = "QlpoOTFBWSZTWQMNh5UAADx7kMkAAIBAAX+AAgBjZB7ABAAAGCAAdQ1T0yQDTTQPUaPKCSUaADQ9QaAbEq4fI1pAWUIsSjpZUgZDKvGcQPCLIeDs6QRMYnLQ4FEXbEKTCiYZ2FQgq1juQ+zDKpuKJfi7kinChIAYbDyo" + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + return data +} + +func createTestTarXz(t *testing.T) []byte { + t.Helper() + buf := new(bytes.Buffer) + w, err := xz.NewWriter(buf) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(createTestTar()); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + func TestTarReader(t *testing.T) { data := createTestTarGz() reader, err := openTar(data, "gzip") @@ -277,6 +313,86 @@ func TestOpen(t *testing.T) { } } +func TestOpenDetectsExtensionlessArchives(t *testing.T) { + tests := []struct { + name string + data []byte + }{ + {"ZIP", createTestZip()}, + {"TAR", createTestTar()}, + {"gzip", createTestTarGz()}, + {"bzip2", createTestTarBz2(t)}, + {"xz", createTestTarXz(t)}, + } + for _, test := range tests { + t.Run(test.name+"/Open", func(t *testing.T) { + reader, err := Open("artifact", bytes.NewReader(test.data)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + files, err := reader.List() + if err != nil { + t.Fatal(err) + } + if len(files) == 0 { + t.Fatal("archive contains no files") + } + }) + + t.Run(test.name+"/OpenBytes", func(t *testing.T) { + reader, err := OpenBytes("artifact", test.data) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + files, err := reader.List() + if err != nil { + t.Fatal(err) + } + if len(files) == 0 { + t.Fatal("archive contains no files") + } + }) + } +} + +func TestOpenKeepsKnownFilenameFormat(t *testing.T) { + _, err := OpenBytes("misleading.tar", createTestZip()) + if err == nil { + t.Fatal("ZIP content with a TAR filename was opened as ZIP") + } + if !strings.Contains(err.Error(), "reading tar") { + t.Fatalf("error = %q, want TAR parser error", err) + } +} + +func TestOpenDoesNotInferGem(t *testing.T) { + reader, err := OpenBytes("artifact", createTestGem()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + if _, ok := reader.(*tarReader); !ok { + t.Fatalf("reader = %T, want generic TAR reader", reader) + } +} + +func TestOpenCompressedNonTar(t *testing.T) { + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + _, _ = w.Write([]byte("not a tar archive")) + _ = w.Close() + + _, err := OpenBytes("artifact", buf.Bytes()) + if err == nil { + t.Fatal("compressed non-TAR content was accepted") + } + if !strings.Contains(err.Error(), "reading tar") { + t.Fatalf("error = %q, want TAR parser error", err) + } +} + func TestZipListDir(t *testing.T) { data := createTestZip() reader, err := openZip(data) diff --git a/hash_bench_test.go b/hash_bench_test.go index 43fa119..a976b6a 100644 --- a/hash_bench_test.go +++ b/hash_bench_test.go @@ -76,3 +76,15 @@ func BenchmarkOpenBytesTarGz(b *testing.B) { _ = r.Close() } } + +func BenchmarkOpenBytesDetectedTarGz(b *testing.B) { + b.SetBytes(int64(len(benchArchive))) + b.ReportAllocs() + for b.Loop() { + r, err := OpenBytes("artifact", benchArchive) + if err != nil { + b.Fatal(err) + } + _ = r.Close() + } +} diff --git a/hash_test.go b/hash_test.go index 3801802..79daf3b 100644 --- a/hash_test.go +++ b/hash_test.go @@ -184,7 +184,7 @@ func TestOpenBytes(t *testing.T) { func TestOpenBytesDoesNotCopy(t *testing.T) { data := createTestZip() - reader, err := OpenBytes("test.zip", data) + reader, err := OpenBytes("artifact", data) if err != nil { t.Fatalf("OpenBytes failed: %v", err) } From f9f2d53500c707c1677e2403e8d5b3faf4dfb874 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 31 Jul 2026 14:20:19 +0100 Subject: [PATCH 2/2] Limit unsupported archive sniffing --- README.md | 3 ++- archives.go | 48 +++++++++++++++++++++++++++++++--------------- archives_test.go | 12 ++++++++++++ hash_bench_test.go | 12 ++++++++++++ 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f4611d5..ecc0b1d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ reader, _ := archives.OpenBytes("pkg.tgz", data) Filename mappings are used first. When a name has no supported extension, `Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, and xz from the content. Compressed content is opened as TAR and returns a parser error when it does not -contain a TAR archive. +contain a TAR archive. `Open` reads at most 512 bytes before rejecting an +unsupported stream with no recognised extension. ### Prefix stripping diff --git a/archives.go b/archives.go index 1748d59..6b10360 100644 --- a/archives.go +++ b/archives.go @@ -10,6 +10,7 @@ package archives import ( + "bufio" "fmt" "io" "path" @@ -20,13 +21,14 @@ import ( ) const ( - formatZIP = "zip" - formatTAR = "tar" - formatTarGzip = "tar.gz" - formatTGZ = "tgz" - formatTarBzip2 = "tar.bz2" - formatTarXZ = "tar.xz" - formatGem = "gem" + formatZIP = "zip" + formatTAR = "tar" + formatTarGzip = "tar.gz" + formatTGZ = "tgz" + formatTarBzip2 = "tar.bz2" + formatTarXZ = "tar.xz" + formatGem = "gem" + contentSniffSize = 512 ) // FileInfo represents metadata about a file in an archive. @@ -66,21 +68,29 @@ type Reader interface { // Open creates an archive reader for the given content. // The filename is used first to detect the archive format. If it has no // supported extension, the content is checked for a supported physical format. -// The content reader will be read entirely into memory. +// Recognised archives are read entirely into memory. An unrecognised stream +// with no supported extension is rejected after reading at most 512 bytes. // //nolint:ireturn // factory function returning interface by design func Open(filename string, content io.Reader) (Reader, error) { format := detectFormat(filename) + if format == "" { + buffered := bufio.NewReaderSize(content, contentSniffSize) + prefix, err := buffered.Peek(contentSniffSize) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("reading archive content: %w", err) + } + format = detectContentPrefixFormat(prefix) + if format == "" { + return nil, fmt.Errorf("unsupported archive format: %s", filename) + } + content = buffered + } + raw, err := io.ReadAll(content) if err != nil { return nil, fmt.Errorf("reading archive content: %w", err) } - if format == "" { - format = detectContentFormat(raw) - } - if format == "" { - return nil, fmt.Errorf("unsupported archive format: %s", filename) - } return openRaw(format, raw) } @@ -123,7 +133,15 @@ func openRaw(format string, raw []byte) (Reader, error) { } func detectContentFormat(content []byte) string { - switch magic.Detect(content).Format { + return archiveFormat(magic.Detect(content).Format) +} + +func detectContentPrefixFormat(content []byte) string { + return archiveFormat(magic.DetectPrefix(content).Format) +} + +func archiveFormat(detected string) string { + switch detected { case "zip": return formatZIP case "tar": diff --git a/archives_test.go b/archives_test.go index 8a26f8c..c343c57 100644 --- a/archives_test.go +++ b/archives_test.go @@ -357,6 +357,18 @@ func TestOpenDetectsExtensionlessArchives(t *testing.T) { } } +func TestOpenLimitsUnsupportedContentRead(t *testing.T) { + content := bytes.NewReader(bytes.Repeat([]byte("x"), contentSniffSize*4)) + _, err := Open("artifact", content) + if err == nil { + t.Fatal("unsupported content was accepted") + } + consumed := content.Size() - int64(content.Len()) + if consumed > contentSniffSize { + t.Fatalf("read %d bytes, want at most %d", consumed, contentSniffSize) + } +} + func TestOpenKeepsKnownFilenameFormat(t *testing.T) { _, err := OpenBytes("misleading.tar", createTestZip()) if err == nil { diff --git a/hash_bench_test.go b/hash_bench_test.go index a976b6a..0609444 100644 --- a/hash_bench_test.go +++ b/hash_bench_test.go @@ -88,3 +88,15 @@ func BenchmarkOpenBytesDetectedTarGz(b *testing.B) { _ = r.Close() } } + +func BenchmarkOpenDetectedTarGz(b *testing.B) { + b.SetBytes(int64(len(benchArchive))) + b.ReportAllocs() + for b.Loop() { + r, err := Open("artifact", bytes.NewReader(benchArchive)) + if err != nil { + b.Fatal(err) + } + _ = r.Close() + } +}