diff --git a/README.md b/README.md index 21f9c52..ecc0b1d 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ The hash is computed over the archive as it was passed to `Open`, not the decomp reader, _ := archives.OpenBytes("pkg.tgz", data) ``` +Filename mappings are used first. When a name has no supported extension, +`Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, and xz from the content. +Compressed content is opened as TAR and returns a parser error when it does not +contain a TAR archive. `Open` reads at most 512 bytes before rejecting an +unsupported stream with no recognised extension. + ### Prefix stripping Some package formats wrap content in a directory (npm uses `package/`). `OpenWithPrefix` strips a path prefix from all entries: diff --git a/archives.go b/archives.go index f7cadc0..6b10360 100644 --- a/archives.go +++ b/archives.go @@ -10,11 +10,25 @@ package archives import ( + "bufio" "fmt" "io" "path" "strings" "time" + + "github.com/git-pkgs/magic" +) + +const ( + formatZIP = "zip" + formatTAR = "tar" + formatTarGzip = "tar.gz" + formatTGZ = "tgz" + formatTarBzip2 = "tar.bz2" + formatTarXZ = "tar.xz" + formatGem = "gem" + contentSniffSize = 512 ) // FileInfo represents metadata about a file in an archive. @@ -52,14 +66,25 @@ type Reader interface { } // Open creates an archive reader for the given content. -// The filename is used to detect the archive format. -// The content reader will be read entirely into memory. +// The filename is used first to detect the archive format. If it has no +// supported extension, the content is checked for a supported physical format. +// Recognised archives are read entirely into memory. An unrecognised stream +// with no supported extension is rejected after reading at most 512 bytes. // //nolint:ireturn // factory function returning interface by design func Open(filename string, content io.Reader) (Reader, error) { format := detectFormat(filename) if format == "" { - return nil, fmt.Errorf("unsupported archive format: %s", filename) + buffered := bufio.NewReaderSize(content, contentSniffSize) + prefix, err := buffered.Peek(contentSniffSize) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("reading archive content: %w", err) + } + format = detectContentPrefixFormat(prefix) + if format == "" { + return nil, fmt.Errorf("unsupported archive format: %s", filename) + } + content = buffered } raw, err := io.ReadAll(content) @@ -77,6 +102,9 @@ func Open(filename string, content io.Reader) (Reader, error) { //nolint:ireturn // factory function returning interface by design func OpenBytes(filename string, content []byte) (Reader, error) { format := detectFormat(filename) + if format == "" { + format = detectContentFormat(content) + } if format == "" { return nil, fmt.Errorf("unsupported archive format: %s", filename) } @@ -87,23 +115,48 @@ func OpenBytes(filename string, content []byte) (Reader, error) { //nolint:ireturn func openRaw(format string, raw []byte) (Reader, error) { switch format { - case "zip": + case formatZIP: return openZip(raw) - case "tar": + case formatTAR: return openTar(raw, "") - case "tar.gz", "tgz": + case formatTarGzip, formatTGZ: return openTar(raw, "gzip") - case "tar.bz2": + case formatTarBzip2: return openTar(raw, "bzip2") - case "tar.xz": + case formatTarXZ: return openTar(raw, "xz") - case "gem": + case formatGem: return openGem(raw) default: return nil, fmt.Errorf("unsupported format: %s", format) } } +func detectContentFormat(content []byte) string { + return archiveFormat(magic.Detect(content).Format) +} + +func detectContentPrefixFormat(content []byte) string { + return archiveFormat(magic.DetectPrefix(content).Format) +} + +func archiveFormat(detected string) string { + switch detected { + case "zip": + return formatZIP + case "tar": + return formatTAR + case "gzip": + return formatTarGzip + case "bzip2": + return formatTarBzip2 + case "xz": + return formatTarXZ + default: + return "" + } +} + // OpenWithPrefix opens an archive and strips the given prefix from all paths. // This is useful for npm packages which wrap content in a "package/" directory. // @@ -143,26 +196,26 @@ func detectFormat(filename string) string { // Check for compound extensions first if strings.HasSuffix(filename, ".tar.gz") { - return "tar.gz" + return formatTarGzip } if strings.HasSuffix(filename, ".tar.bz2") { - return "tar.bz2" + return formatTarBzip2 } if strings.HasSuffix(filename, ".tar.xz") { - return "tar.xz" + return formatTarXZ } // Check simple extensions ext := path.Ext(filename) switch ext { case ".zip", ".jar", ".whl", ".nupkg", ".egg": - return "zip" + return formatZIP case ".tar": - return "tar" + return formatTAR case ".tgz": - return "tgz" + return formatTGZ case ".gem": - return "gem" + return formatGem default: return "" } diff --git a/archives_test.go b/archives_test.go index bbdf59e..c343c57 100644 --- a/archives_test.go +++ b/archives_test.go @@ -5,6 +5,7 @@ import ( "archive/zip" "bytes" "compress/gzip" + "encoding/base64" "errors" "fmt" "io" @@ -12,6 +13,8 @@ import ( "strings" "testing" "time" + + "github.com/ulikunitz/xz" ) func TestDetectFormat(t *testing.T) { @@ -183,11 +186,9 @@ func TestZipReader(t *testing.T) { } } -// createTestTarGz creates a tar.gz archive in memory with test files -func createTestTarGz() []byte { +func createTestTar() []byte { buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - tw := tar.NewWriter(gw) + tw := tar.NewWriter(buf) files := []struct { name string @@ -210,10 +211,45 @@ func createTestTarGz() []byte { } _ = tw.Close() + return buf.Bytes() +} + +// createTestTarGz creates a tar.gz archive in memory with test files +func createTestTarGz() []byte { + buf := new(bytes.Buffer) + gw := gzip.NewWriter(buf) + _, _ = gw.Write(createTestTar()) _ = gw.Close() return buf.Bytes() } +func createTestTarBz2(t *testing.T) []byte { + t.Helper() + // The standard library provides a bzip2 reader but no writer. + const encoded = "QlpoOTFBWSZTWQMNh5UAADx7kMkAAIBAAX+AAgBjZB7ABAAAGCAAdQ1T0yQDTTQPUaPKCSUaADQ9QaAbEq4fI1pAWUIsSjpZUgZDKvGcQPCLIeDs6QRMYnLQ4FEXbEKTCiYZ2FQgq1juQ+zDKpuKJfi7kinChIAYbDyo" + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + return data +} + +func createTestTarXz(t *testing.T) []byte { + t.Helper() + buf := new(bytes.Buffer) + w, err := xz.NewWriter(buf) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(createTestTar()); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + func TestTarReader(t *testing.T) { data := createTestTarGz() reader, err := openTar(data, "gzip") @@ -277,6 +313,98 @@ func TestOpen(t *testing.T) { } } +func TestOpenDetectsExtensionlessArchives(t *testing.T) { + tests := []struct { + name string + data []byte + }{ + {"ZIP", createTestZip()}, + {"TAR", createTestTar()}, + {"gzip", createTestTarGz()}, + {"bzip2", createTestTarBz2(t)}, + {"xz", createTestTarXz(t)}, + } + for _, test := range tests { + t.Run(test.name+"/Open", func(t *testing.T) { + reader, err := Open("artifact", bytes.NewReader(test.data)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + files, err := reader.List() + if err != nil { + t.Fatal(err) + } + if len(files) == 0 { + t.Fatal("archive contains no files") + } + }) + + t.Run(test.name+"/OpenBytes", func(t *testing.T) { + reader, err := OpenBytes("artifact", test.data) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + files, err := reader.List() + if err != nil { + t.Fatal(err) + } + if len(files) == 0 { + t.Fatal("archive contains no files") + } + }) + } +} + +func TestOpenLimitsUnsupportedContentRead(t *testing.T) { + content := bytes.NewReader(bytes.Repeat([]byte("x"), contentSniffSize*4)) + _, err := Open("artifact", content) + if err == nil { + t.Fatal("unsupported content was accepted") + } + consumed := content.Size() - int64(content.Len()) + if consumed > contentSniffSize { + t.Fatalf("read %d bytes, want at most %d", consumed, contentSniffSize) + } +} + +func TestOpenKeepsKnownFilenameFormat(t *testing.T) { + _, err := OpenBytes("misleading.tar", createTestZip()) + if err == nil { + t.Fatal("ZIP content with a TAR filename was opened as ZIP") + } + if !strings.Contains(err.Error(), "reading tar") { + t.Fatalf("error = %q, want TAR parser error", err) + } +} + +func TestOpenDoesNotInferGem(t *testing.T) { + reader, err := OpenBytes("artifact", createTestGem()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reader.Close() }() + if _, ok := reader.(*tarReader); !ok { + t.Fatalf("reader = %T, want generic TAR reader", reader) + } +} + +func TestOpenCompressedNonTar(t *testing.T) { + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + _, _ = w.Write([]byte("not a tar archive")) + _ = w.Close() + + _, err := OpenBytes("artifact", buf.Bytes()) + if err == nil { + t.Fatal("compressed non-TAR content was accepted") + } + if !strings.Contains(err.Error(), "reading tar") { + t.Fatalf("error = %q, want TAR parser error", err) + } +} + func TestZipListDir(t *testing.T) { data := createTestZip() reader, err := openZip(data) diff --git a/hash_bench_test.go b/hash_bench_test.go index 43fa119..0609444 100644 --- a/hash_bench_test.go +++ b/hash_bench_test.go @@ -76,3 +76,27 @@ func BenchmarkOpenBytesTarGz(b *testing.B) { _ = r.Close() } } + +func BenchmarkOpenBytesDetectedTarGz(b *testing.B) { + b.SetBytes(int64(len(benchArchive))) + b.ReportAllocs() + for b.Loop() { + r, err := OpenBytes("artifact", benchArchive) + if err != nil { + b.Fatal(err) + } + _ = r.Close() + } +} + +func BenchmarkOpenDetectedTarGz(b *testing.B) { + b.SetBytes(int64(len(benchArchive))) + b.ReportAllocs() + for b.Loop() { + r, err := Open("artifact", bytes.NewReader(benchArchive)) + if err != nil { + b.Fatal(err) + } + _ = r.Close() + } +} diff --git a/hash_test.go b/hash_test.go index 3801802..79daf3b 100644 --- a/hash_test.go +++ b/hash_test.go @@ -184,7 +184,7 @@ func TestOpenBytes(t *testing.T) { func TestOpenBytesDoesNotCopy(t *testing.T) { data := createTestZip() - reader, err := OpenBytes("test.zip", data) + reader, err := OpenBytes("artifact", data) if err != nil { t.Fatalf("OpenBytes failed: %v", err) }