Detect extensionless archive formats

README.md

@@ -66,6 +66,12 @@ The hash is computed over the archive as it was passed to `Open`, not the decomp
 reader, _ := archives.OpenBytes("pkg.tgz", data)
 ```
 
+Filename mappings are used first. When a name has no supported extension,
+`Open` and `OpenBytes` detect ZIP, TAR, gzip, bzip2, and xz from the content.
+Compressed content is opened as TAR and returns a parser error when it does not
+contain a TAR archive. `Open` reads at most 512 bytes before rejecting an
+unsupported stream with no recognised extension.
+
 ### Prefix stripping
 
 Some package formats wrap content in a directory (npm uses `package/`). `OpenWithPrefix` strips a path prefix from all entries:

archives.go

@@ -10,11 +10,25 @@
 package archives
 
 import (
+	"bufio"
 	"fmt"
 	"io"
 	"path"
 	"strings"
 	"time"
+
+	"github.com/git-pkgs/magic"
+)
+
+const (
+	formatZIP        = "zip"
+	formatTAR        = "tar"
+	formatTarGzip    = "tar.gz"
+	formatTGZ        = "tgz"
+	formatTarBzip2   = "tar.bz2"
+	formatTarXZ      = "tar.xz"
+	formatGem        = "gem"
+	contentSniffSize = 512
 )
 
 // FileInfo represents metadata about a file in an archive.
@@ -52,14 +66,25 @@ type Reader interface {
 }
 
 // Open creates an archive reader for the given content.
-// The filename is used to detect the archive format.
-// The content reader will be read entirely into memory.
+// The filename is used first to detect the archive format. If it has no
+// supported extension, the content is checked for a supported physical format.
+// Recognised archives are read entirely into memory. An unrecognised stream
+// with no supported extension is rejected after reading at most 512 bytes.
 //
 //nolint:ireturn // factory function returning interface by design
 func Open(filename string, content io.Reader) (Reader, error) {
 	format := detectFormat(filename)
 	if format == "" {
-		return nil, fmt.Errorf("unsupported archive format: %s", filename)
+		buffered := bufio.NewReaderSize(content, contentSniffSize)
+		prefix, err := buffered.Peek(contentSniffSize)
+		if err != nil && err != io.EOF {
+			return nil, fmt.Errorf("reading archive content: %w", err)
+		}
+		format = detectContentPrefixFormat(prefix)
+		if format == "" {
+			return nil, fmt.Errorf("unsupported archive format: %s", filename)
+		}
+		content = buffered
 	}
 
 	raw, err := io.ReadAll(content)
@@ -77,6 +102,9 @@ func Open(filename string, content io.Reader) (Reader, error) {
 //nolint:ireturn // factory function returning interface by design
 func OpenBytes(filename string, content []byte) (Reader, error) {
 	format := detectFormat(filename)
+	if format == "" {
+		format = detectContentFormat(content)
+	}
 	if format == "" {
 		return nil, fmt.Errorf("unsupported archive format: %s", filename)
 	}
@@ -87,23 +115,48 @@ func OpenBytes(filename string, content []byte) (Reader, error) {
 //nolint:ireturn
 func openRaw(format string, raw []byte) (Reader, error) {
 	switch format {
-	case "zip":
+	case formatZIP:
 		return openZip(raw)
-	case "tar":
+	case formatTAR:
 		return openTar(raw, "")
-	case "tar.gz", "tgz":
+	case formatTarGzip, formatTGZ:
 		return openTar(raw, "gzip")
-	case "tar.bz2":
+	case formatTarBzip2:
 		return openTar(raw, "bzip2")
-	case "tar.xz":
+	case formatTarXZ:
 		return openTar(raw, "xz")
-	case "gem":
+	case formatGem:
 		return openGem(raw)
 	default:
 		return nil, fmt.Errorf("unsupported format: %s", format)
 	}
 }
 
+func detectContentFormat(content []byte) string {
+	return archiveFormat(magic.Detect(content).Format)
+}
+
+func detectContentPrefixFormat(content []byte) string {
+	return archiveFormat(magic.DetectPrefix(content).Format)
+}
+
+func archiveFormat(detected string) string {
+	switch detected {
+	case "zip":
+		return formatZIP
+	case "tar":
+		return formatTAR
+	case "gzip":
+		return formatTarGzip
+	case "bzip2":
+		return formatTarBzip2
+	case "xz":
+		return formatTarXZ
+	default:
+		return ""
+	}
+}
+
 // OpenWithPrefix opens an archive and strips the given prefix from all paths.
 // This is useful for npm packages which wrap content in a "package/" directory.
 //
@@ -143,26 +196,26 @@ func detectFormat(filename string) string {
 
 	// Check for compound extensions first
 	if strings.HasSuffix(filename, ".tar.gz") {
-		return "tar.gz"
+		return formatTarGzip
 	}
 	if strings.HasSuffix(filename, ".tar.bz2") {
-		return "tar.bz2"
+		return formatTarBzip2
 	}
 	if strings.HasSuffix(filename, ".tar.xz") {
-		return "tar.xz"
+		return formatTarXZ
 	}
 
 	// Check simple extensions
 	ext := path.Ext(filename)
 	switch ext {
 	case ".zip", ".jar", ".whl", ".nupkg", ".egg":
-		return "zip"
+		return formatZIP
 	case ".tar":
-		return "tar"
+		return formatTAR
 	case ".tgz":
-		return "tgz"
+		return formatTGZ
 	case ".gem":
-		return "gem"
+		return formatGem
 	default:
 		return ""
 	}

archives_test.go

@@ -5,13 +5,16 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/ulikunitz/xz"
 )
 
 func TestDetectFormat(t *testing.T) {
@@ -183,11 +186,9 @@ func TestZipReader(t *testing.T) {
 	}
 }
 
-// createTestTarGz creates a tar.gz archive in memory with test files
-func createTestTarGz() []byte {
+func createTestTar() []byte {
 	buf := new(bytes.Buffer)
-	gw := gzip.NewWriter(buf)
-	tw := tar.NewWriter(gw)
+	tw := tar.NewWriter(buf)
 
 	files := []struct {
 		name    string
@@ -210,10 +211,45 @@ func createTestTarGz() []byte {
 	}
 
 	_ = tw.Close()
+	return buf.Bytes()
+}
+
+// createTestTarGz creates a tar.gz archive in memory with test files
+func createTestTarGz() []byte {
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	_, _ = gw.Write(createTestTar())
 	_ = gw.Close()
 	return buf.Bytes()
 }
 
+func createTestTarBz2(t *testing.T) []byte {
+	t.Helper()
+	// The standard library provides a bzip2 reader but no writer.
+	const encoded = "QlpoOTFBWSZTWQMNh5UAADx7kMkAAIBAAX+AAgBjZB7ABAAAGCAAdQ1T0yQDTTQPUaPKCSUaADQ9QaAbEq4fI1pAWUIsSjpZUgZDKvGcQPCLIeDs6QRMYnLQ4FEXbEKTCiYZ2FQgq1juQ+zDKpuKJfi7kinChIAYbDyo"
+	data, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func createTestTarXz(t *testing.T) []byte {
+	t.Helper()
+	buf := new(bytes.Buffer)
+	w, err := xz.NewWriter(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := w.Write(createTestTar()); err != nil {
+		t.Fatal(err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
 func TestTarReader(t *testing.T) {
 	data := createTestTarGz()
 	reader, err := openTar(data, "gzip")
@@ -277,6 +313,98 @@ func TestOpen(t *testing.T) {
 	}
 }
 
+func TestOpenDetectsExtensionlessArchives(t *testing.T) {
+	tests := []struct {
+		name string
+		data []byte
+	}{
+		{"ZIP", createTestZip()},
+		{"TAR", createTestTar()},
+		{"gzip", createTestTarGz()},
+		{"bzip2", createTestTarBz2(t)},
+		{"xz", createTestTarXz(t)},
+	}
+	for _, test := range tests {
+		t.Run(test.name+"/Open", func(t *testing.T) {
+			reader, err := Open("artifact", bytes.NewReader(test.data))
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() { _ = reader.Close() }()
+			files, err := reader.List()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(files) == 0 {
+				t.Fatal("archive contains no files")
+			}
+		})
+
+		t.Run(test.name+"/OpenBytes", func(t *testing.T) {
+			reader, err := OpenBytes("artifact", test.data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer func() { _ = reader.Close() }()
+			files, err := reader.List()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(files) == 0 {
+				t.Fatal("archive contains no files")
+			}
+		})
+	}
+}
+
+func TestOpenLimitsUnsupportedContentRead(t *testing.T) {
+	content := bytes.NewReader(bytes.Repeat([]byte("x"), contentSniffSize*4))
+	_, err := Open("artifact", content)
+	if err == nil {
+		t.Fatal("unsupported content was accepted")
+	}
+	consumed := content.Size() - int64(content.Len())
+	if consumed > contentSniffSize {
+		t.Fatalf("read %d bytes, want at most %d", consumed, contentSniffSize)
+	}
+}
+
+func TestOpenKeepsKnownFilenameFormat(t *testing.T) {
+	_, err := OpenBytes("misleading.tar", createTestZip())
+	if err == nil {
+		t.Fatal("ZIP content with a TAR filename was opened as ZIP")
+	}
+	if !strings.Contains(err.Error(), "reading tar") {
+		t.Fatalf("error = %q, want TAR parser error", err)
+	}
+}
+
+func TestOpenDoesNotInferGem(t *testing.T) {
+	reader, err := OpenBytes("artifact", createTestGem())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = reader.Close() }()
+	if _, ok := reader.(*tarReader); !ok {
+		t.Fatalf("reader = %T, want generic TAR reader", reader)
+	}
+}
+
+func TestOpenCompressedNonTar(t *testing.T) {
+	buf := new(bytes.Buffer)
+	w := gzip.NewWriter(buf)
+	_, _ = w.Write([]byte("not a tar archive"))
+	_ = w.Close()
+
+	_, err := OpenBytes("artifact", buf.Bytes())
+	if err == nil {
+		t.Fatal("compressed non-TAR content was accepted")
+	}
+	if !strings.Contains(err.Error(), "reading tar") {
+		t.Fatalf("error = %q, want TAR parser error", err)
+	}
+}
+
 func TestZipListDir(t *testing.T) {
 	data := createTestZip()
 	reader, err := openZip(data)

hash_bench_test.go

@@ -76,3 +76,27 @@ func BenchmarkOpenBytesTarGz(b *testing.B) {
 		_ = r.Close()
 	}
 }
+
+func BenchmarkOpenBytesDetectedTarGz(b *testing.B) {
+	b.SetBytes(int64(len(benchArchive)))
+	b.ReportAllocs()
+	for b.Loop() {
+		r, err := OpenBytes("artifact", benchArchive)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_ = r.Close()
+	}
+}
+
+func BenchmarkOpenDetectedTarGz(b *testing.B) {
+	b.SetBytes(int64(len(benchArchive)))
+	b.ReportAllocs()
+	for b.Loop() {
+		r, err := Open("artifact", bytes.NewReader(benchArchive))
+		if err != nil {
+			b.Fatal(err)
+		}
+		_ = r.Close()
+	}
+}

hash_test.go

@@ -184,7 +184,7 @@ func TestOpenBytes(t *testing.T) {
 
 func TestOpenBytesDoesNotCopy(t *testing.T) {
 	data := createTestZip()
-	reader, err := OpenBytes("test.zip", data)
+	reader, err := OpenBytes("artifact", data)
 	if err != nil {
 		t.Fatalf("OpenBytes failed: %v", err)
 	}