diff --git a/src/Archive/Archive.jl b/src/Archive/Archive.jl new file mode 100644 index 0000000..716d55a --- /dev/null +++ b/src/Archive/Archive.jl @@ -0,0 +1,354 @@ +module Archive + +using ObjectFile, StructIO +import ObjectFile: readmeta, path +import Base: show, read, getindex, iterate, length, keys, firstindex, lastindex, eltype + +export ArchiveHandle, ArchiveMember, ArchiveMemberHeader, archive_kind, isthin + +# Each archive starts with one of these, followed by a sequence of members +const AR_MAGIC = b"!\n" +const AR_THIN_MAGIC = b"!\n" + +# Each member starts with an `ArchiveMemberHeader` ending in `AR_FMAG` +const AR_FMAG = b"`\n" + +# The members that index an archive rather than being a part of it, by kind of +# archive (see `archive_kind()`) +const GNU_SYMTAB = "/" +const GNU_SYMTAB64 = "/SYM64/" +const GNU_NAMETAB = "//" +const COFF_EC_SYMTAB = "//" +const BSD_SYMTAB = ("__.SYMDEF", "__.SYMDEF SORTED") +const BSD_SYMTAB64 = ("__.SYMDEF_64", "__.SYMDEF_64 SORTED") +const BSD_LONG_NAME_PREFIX = "#1/" + +""" + ArchiveMemberHeader + +The header of a member of a static archive (`ar_hdr`), made of space-padded +ASCII fields. `name` is the member's name, as encoded by the kind of archive (see +`archive_kind()`), `mode` is in octal, and the other fields are in decimal. +""" +@io struct ArchiveMemberHeader + name::NTuple{16,UInt8} + mtime::NTuple{12,UInt8} + uid::NTuple{6,UInt8} + gid::NTuple{6,UInt8} + mode::NTuple{8,UInt8} + size::NTuple{10,UInt8} + fmag::NTuple{2,UInt8} +end align_packed + +field_string(field::NTuple{N,UInt8}) where {N} = rstrip(String(collect(field)), ' ') + +# Parse an integer field of a member header, which may be blank +function parse_field(field::NTuple{N,UInt8}, base::Int = 10) where {N} + s = field_string(field) + return isempty(s) ? 0 : parse(Int64, s; base) +end + +""" + ArchiveMember + +A member of a static archive, as listed in an [`ArchiveHandle`](@ref). `offset` +is the location of the member's data relative to the start of the archive, or +`nothing` for members of a thin archive, whose data lives in the file `name` +(relative to the archive). +""" +struct ArchiveMember + name::String + offset::Union{Nothing,Int64} + size::Int64 + mtime::Int64 + uid::Int64 + gid::Int64 + mode::UInt32 +end + +""" + ArchiveHandle + +A handle on a static archive (an `ar` archive, e.g. `libfoo.a` or `foo.lib`). +Iterate over it to get the [`ArchiveMember`](@ref)s, then `readmeta(ah, member)` +to open one as an object file, or `read(ah, member)` to get its contents. + +The `ar` format comes in several kinds (see [`archive_kind()`](@ref)), which +differ in how they name members and index symbols. GNU thin archives, whose +members are stored outside of the archive, are supported too (see `isthin()`). + +Archives are not opened by `readmeta(io)`, as they are a collection of object +files, not one. Use `readmeta(io, ArchiveHandle)` instead. +""" +struct ArchiveHandle{T<:IO} + # Backing IO and start point within the IOStream of this archive + io::T + start::Int64 + + # The kind of archive, see `archive_kind()` + kind::Symbol + + # Whether this is a GNU thin archive, whose members are stored outside of it + thin::Bool + + # Members, in archive order, excluding symbol and name tables + members::Vector{ArchiveMember} + + # The path of the file this was created with, if it exists + path::String +end + +# A member as framed by its header within the archive, before its name (which +# may refer to data elsewhere, see `member_name()`) is resolved +struct RawMember + header_offset::Int64 + header::ArchiveMemberHeader + data_offset::Int64 + size::Int64 +end + +# The member's name field, without its padding +name_field(raw::RawMember) = field_string(raw.header.name) + +function malformed(raw::RawMember, msg) + throw(ArgumentError("Malformed archive member at offset $(raw.header_offset): $(msg)")) +end + +function read_raw_members(io::IO, start::Integer, thin::Bool) + raw = RawMember[] + while !eof(io) + header_offset = position(io) - start + header = unpack(io, ArchiveMemberHeader) + if collect(header.fmag) != AR_FMAG + throw(ArgumentError("Malformed archive member header at offset $(header_offset)")) + end + data_offset = position(io) - start + size = parse_field(header.size) + push!(raw, RawMember(header_offset, header, data_offset, size)) + + # Skip to the next header, which is aligned to an even offset. A thin + # archive (which is always of the GNU kind) only stores the members that + # index it within itself. + if !thin || is_index_member(:gnu, field_string(header.name)) + next_offset = data_offset + size + seek(io, start + next_offset + isodd(next_offset)) + end + end + return raw +end + +is_gnu(kind::Symbol) = kind in (:gnu, :gnu64) +is_bsd(kind::Symbol) = kind in (:bsd, :bsd64) + +# Whether `name` is that of a member which indexes an archive of the given kind +# (a symbol or long name table), rather than being a part of it +function is_index_member(kind::Symbol, name::AbstractString) + if is_bsd(kind) + return name in BSD_SYMTAB || name in BSD_SYMTAB64 + end + return name in (GNU_SYMTAB, GNU_SYMTAB64, GNU_NAMETAB, COFF_EC_SYMTAB) +end + +# Look up a name stored at `offset` in the long name table. GNU terminates names +# with "/\n", COFF with a NUL. +function lookup_long_name(raw::RawMember, kind::Symbol, table::Union{Nothing,Vector{UInt8}}) + field = name_field(raw) + offset = tryparse(Int, field[2:end]) + if table === nothing + malformed(raw, "long name $(repr(field)) without a long name table") + elseif offset === nothing || !(0 <= offset < length(table)) + malformed(raw, "long name $(repr(field)) is outside of the long name table") + end + terminator = kind == :coff ? 0x00 : UInt8('\n') + stop = findnext(==(terminator), table, offset + 1) + if stop === nothing + malformed(raw, "long name $(repr(field)) is not terminated") + end + name = String(table[offset+1:stop-1]) + if kind != :coff + endswith(name, "/") || malformed(raw, "long name $(repr(name)) is not terminated by \"/\"") + name = name[1:end-1] + end + return name +end + +""" + member_name(io, start, raw, kind, long_names) + +Resolve the name of the member `raw` of an archive (starting at `start` within +`io`) of the given `kind`, returning it along with how many bytes at the start of +the member's data it takes up: + + - BSD stores names that are long (or contain spaces) at the start of the + member's data, naming the member `#1/`. + - GNU and COFF refer to long names as `/` into the `long_names` table, + and terminate short names with a "/" (to allow for trailing spaces). The + members that index the archive have reserved names (e.g. `/` and `//`), + which are returned as-is. +""" +function member_name(io::IO, start::Integer, raw::RawMember, kind::Symbol, long_names) + field = name_field(raw) + if is_bsd(kind) + startswith(field, BSD_LONG_NAME_PREFIX) || return field, 0 + name_size = tryparse(Int, field[length(BSD_LONG_NAME_PREFIX)+1:end]) + if name_size === nothing || !(0 <= name_size <= raw.size) + malformed(raw, "invalid BSD long name $(repr(field))") + end + seek(io, start + raw.data_offset) + return rstrip(String(read(io, name_size)), '\0'), name_size + end + + if is_index_member(kind, field) + return field, 0 + elseif startswith(field, "/") + return lookup_long_name(raw, kind, long_names), 0 + elseif endswith(field, "/") + return field[1:end-1], 0 + end + malformed(raw, "member name $(repr(field)) is not terminated by \"/\"") +end + +# Determine the kind of archive from its first member(s), along with how many of +# them are symbol tables +function detect_kind(io::IO, start::Integer, raw::Vector{RawMember}) + # An empty archive is the same in every kind + isempty(raw) && return :gnu, 0 + + field = name_field(raw[1]) + if field == GNU_SYMTAB + # COFF archives start with two linker members, both named like a GNU symbol table + return length(raw) > 1 && name_field(raw[2]) == GNU_SYMTAB ? (:coff, 2) : (:gnu, 1) + elseif field == GNU_SYMTAB64 + return :gnu64, 1 + elseif startswith(field, "/") || endswith(field, "/") + # Without a symbol table, a GNU archive starts with its name table ("//") + # or a member, whose names are always terminated (or referred to) by a "/". + return :gnu, 0 + end + + # Otherwise this is a BSD archive, possibly starting with its symbol table + name = first(member_name(io, start, raw[1], :bsd, nothing)) + return name in BSD_SYMTAB64 ? (:bsd64, 1) : (:bsd, Int(name in BSD_SYMTAB)) +end + +function readmeta(io::IO, ::Type{ArchiveHandle}) + start = position(io) + magic = read(io, length(AR_MAGIC)) + thin = magic == AR_THIN_MAGIC + if magic != AR_MAGIC && !thin + throw(MagicMismatch("Magic Number $(repr(String(magic))) does not match expected archive magic number $(repr(String(AR_MAGIC)))")) + end + + raw = read_raw_members(io, start, thin) + kind, num_symtabs = detect_kind(io, start, raw) + if thin && !is_gnu(kind) + throw(ArgumentError("Malformed archive: thin archives must be of the GNU kind, not $(kind)")) + end + + # Skip over the members that index the archive, which come first: the symbol + # table(s), then (for GNU and COFF) an optional long name table, then (for + # COFF) an optional ARM64EC symbol table. + idx = 1 + num_symtabs + long_names = nothing + if !is_bsd(kind) && idx <= length(raw) && name_field(raw[idx]) == GNU_NAMETAB + seek(io, start + raw[idx].data_offset) + long_names = read(io, raw[idx].size) + idx += 1 + end + if kind == :coff && idx <= length(raw) && name_field(raw[idx]) == COFF_EC_SYMTAB + idx += 1 + end + + members = ArchiveMember[] + for r in raw[idx:end] + name, name_size = member_name(io, start, r, kind, long_names) + if is_index_member(kind, name) + malformed(r, "unexpected $(repr(name)) member in a $(kind) archive") + end + push!(members, ArchiveMember( + name, + thin ? nothing : r.data_offset + name_size, + r.size - name_size, + parse_field(r.header.mtime), + parse_field(r.header.uid), + parse_field(r.header.gid), + UInt32(parse_field(r.header.mode, 8)), + )) + end + + return ArchiveHandle(io, Int64(start), kind, thin, members, path(io)) +end + +""" + archive_kind(ah::ArchiveHandle) + +Return the kind of the archive `ah`, which determines how it names its members +and indexes their symbols: + + - `:gnu`: System V / GNU archives, with a `/` symbol table, and a `//` table of + long names (referred to as `/`) + - `:gnu64`: GNU archives with a 64-bit (`/SYM64/`) symbol table + - `:bsd`: BSD (incl. Darwin) archives, with a `__.SYMDEF` symbol table, and long + names stored before the member's data (named `#1/`) + - `:bsd64`: BSD archives with a 64-bit (`__.SYMDEF_64`) symbol table + - `:coff`: COFF (e.g. MSVC) archives, which start with two `/` linker members, + and have GNU-like names (but NUL-terminated long names) + +Archives without a symbol table are told apart by their member names, as only +GNU (and COFF) names are terminated by a `/`. +""" +archive_kind(ah::ArchiveHandle) = ah.kind + +""" + isthin(ah::ArchiveHandle) + +Returns `true` if `ah` is a GNU thin archive, whose members are not stored within +the archive, but referred to by their path relative to it. +""" +isthin(ah::ArchiveHandle) = ah.thin + +path(ah::ArchiveHandle) = ah.path + +# An archive is an indexable collection of its members +keys(ah::ArchiveHandle) = keys(ah.members) +iterate(ah::ArchiveHandle, idx=1) = iterate(ah.members, idx) +length(ah::ArchiveHandle) = length(ah.members) +firstindex(ah::ArchiveHandle) = firstindex(ah.members) +lastindex(ah::ArchiveHandle) = lastindex(ah.members) +eltype(::Type{<:ArchiveHandle}) = ArchiveMember +getindex(ah::ArchiveHandle, idx) = ah.members[idx] + +# Seek to the data of `member`, which must be stored within the archive +function seek_member(ah::ArchiveHandle, member::ArchiveMember) + if member.offset === nothing + throw(ArgumentError("Member $(member.name) of thin archive $(ah.path) is stored outside of the archive")) + end + seek(ah.io, ah.start + member.offset) +end + +""" + read(ah::ArchiveHandle, member::ArchiveMember) + +Read the contents of `member` out of the archive `ah`. +""" +function read(ah::ArchiveHandle, member::ArchiveMember) + seek_member(ah, member) + return read(ah.io, member.size) +end + +""" + readmeta(ah::ArchiveHandle, member::ArchiveMember) + +Read the object file stored as `member` of the archive `ah`, as `readmeta(io)` +would. The resulting handles share their IO with `ah`. +""" +function readmeta(ah::ArchiveHandle, member::ArchiveMember) + seek_member(ah, member) + return readmeta(ah.io) +end + +function show(io::IO, ah::ArchiveHandle) + print(io, "Archive Handle ($(archive_kind(ah)), $(length(ah)) members$(isthin(ah) ? ", thin" : ""))") +end + +end # module Archive diff --git a/src/ObjectFile.jl b/src/ObjectFile.jl index 6ef1bb8..254234a 100644 --- a/src/ObjectFile.jl +++ b/src/ObjectFile.jl @@ -21,6 +21,10 @@ include("MachO/MachO.jl") include("COFF/COFF.jl") @reexport using .COFF +# Include static archives (`ar`), which hold object files of the above formats +include("Archive/Archive.jl") +@reexport using .Archive + function __init__() global ObjTypes @@ -28,6 +32,8 @@ function __init__() push!(ObjTypes, MachOHandle) push!(ObjTypes, FatMachOHandle) push!(ObjTypes, COFFHandle) + # `ArchiveHandle` is deliberately not registered: `readmeta(io)` returns + # object files, and an archive is a collection of them. end end #module ObjectFile diff --git a/test/archives/build.sh b/test/archives/build.sh new file mode 100755 index 0000000..32bdbe8 --- /dev/null +++ b/test/archives/build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Regenerates the static archive fixtures in this directory. Requires `clang`, +# `llvm-ar`, `llvm-lib` and `llvm-dlltool`. The object file names are chosen to +# exercise both short member names and those that need a long name table. +set -euo pipefail +cd "$(dirname "$0")" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +echo 'int foo(int x) { return x + 1; }' > "${tmp}/foo.c" +echo 'int foo(int x); int bar(int x) { return foo(x) * 2; }' > "${tmp}/bar_with_a_long_name.c" + +compile() { + for src in foo bar_with_a_long_name; do + clang --target="$@" -O2 -c -o "${tmp}/${src}.o" "${tmp}/${src}.c" + done +} +# `archive [ops]`, where `ops` defaults to `rcsD` (with a symbol table) +archive() { + local format="$1" out="$2" ops="${3:-rcsD}" + rm -f "${out}" + (cd "${tmp}" && llvm-ar --format="${format}" "${ops}" "${OLDPWD}/${out}" foo.o bar_with_a_long_name.o) +} + +compile x86_64-linux-gnu +archive gnu libfoo_gnu.a +archive gnu libfoo_gnu_nosymtab.a rcSD +# (`SYM64_THRESHOLD` forces a 64-bit symbol table) +SYM64_THRESHOLD=0 archive gnu libfoo_gnu64.a +# Some archivers record member paths relative to the archive, so build those beside them +(cd "${tmp}" && llvm-ar --format=gnu --thin rcsD libfoo_thin.a foo.o bar_with_a_long_name.o) +mv "${tmp}/libfoo_thin.a" libfoo_thin.a + +compile x86_64-apple-macos +archive darwin libfoo_darwin.a +SYM64_THRESHOLD=0 archive darwin libfoo_darwin64.a +archive bsd libfoo_bsd_nosymtab.a rcSD + +# (`-mno-incremental-linker-compatible` keeps timestamps out of COFF objects) +compile x86_64-pc-windows-msvc -mno-incremental-linker-compatible +(cd "${tmp}" && llvm-lib /out:foo.lib foo.o bar_with_a_long_name.o) +mv "${tmp}/foo.lib" foo.lib + +# An import library: an archive too, but of short import objects +printf 'LIBRARY foo.dll\nEXPORTS\nfoo\nbar\n' > "${tmp}/foo.def" +rm -f libfoo.dll.a +llvm-dlltool -m i386:x86-64 -d "${tmp}/foo.def" -l libfoo.dll.a diff --git a/test/archives/foo.lib b/test/archives/foo.lib new file mode 100644 index 0000000..269733b Binary files /dev/null and b/test/archives/foo.lib differ diff --git a/test/archives/libfoo.dll.a b/test/archives/libfoo.dll.a new file mode 100644 index 0000000..e65e858 Binary files /dev/null and b/test/archives/libfoo.dll.a differ diff --git a/test/archives/libfoo_bsd_nosymtab.a b/test/archives/libfoo_bsd_nosymtab.a new file mode 100644 index 0000000..c385379 Binary files /dev/null and b/test/archives/libfoo_bsd_nosymtab.a differ diff --git a/test/archives/libfoo_darwin.a b/test/archives/libfoo_darwin.a new file mode 100644 index 0000000..a00ed9a Binary files /dev/null and b/test/archives/libfoo_darwin.a differ diff --git a/test/archives/libfoo_darwin64.a b/test/archives/libfoo_darwin64.a new file mode 100644 index 0000000..d3a7b3f Binary files /dev/null and b/test/archives/libfoo_darwin64.a differ diff --git a/test/archives/libfoo_gnu.a b/test/archives/libfoo_gnu.a new file mode 100644 index 0000000..257939e Binary files /dev/null and b/test/archives/libfoo_gnu.a differ diff --git a/test/archives/libfoo_gnu64.a b/test/archives/libfoo_gnu64.a new file mode 100644 index 0000000..1e473d0 Binary files /dev/null and b/test/archives/libfoo_gnu64.a differ diff --git a/test/archives/libfoo_gnu_nosymtab.a b/test/archives/libfoo_gnu_nosymtab.a new file mode 100644 index 0000000..2b66a76 Binary files /dev/null and b/test/archives/libfoo_gnu_nosymtab.a differ diff --git a/test/archives/libfoo_thin.a b/test/archives/libfoo_thin.a new file mode 100644 index 0000000..b0c0dae Binary files /dev/null and b/test/archives/libfoo_thin.a differ diff --git a/test/runtests.jl b/test/runtests.jl index c5b1146..3630586 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -337,3 +337,131 @@ using Mmap find_dep_libs(joinpath("./libjulias", file)) end end + +# Fixtures are generated by `./archives/build.sh` +@testset "Archives" begin + member_names = ["foo.o", "bar_with_a_long_name.o"] + + @testset "$(file)" for (file, kind, H, names) in ( + ("libfoo_gnu.a", :gnu, ELFHandle, member_names), + ("libfoo_gnu_nosymtab.a", :gnu, ELFHandle, member_names), + ("libfoo_gnu64.a", :gnu64, ELFHandle, member_names), + ("libfoo_darwin.a", :bsd, MachOHandle, member_names), + ("libfoo_darwin64.a", :bsd64, MachOHandle, member_names), + ("libfoo_bsd_nosymtab.a", :bsd, MachOHandle, member_names), + # `llvm-lib` orders members by name + ("foo.lib", :coff, nothing, reverse(member_names)), + ) + ah = readmeta(open(joinpath("./archives", file), "r"), ArchiveHandle) + @test archive_kind(ah) == kind + @test !isthin(ah) + @test length(ah) == 2 + @test eltype(ah) == ArchiveMember + # Symbol and name tables are not listed as members + @test [m.name for m in ah] == names + # An archive is an indexable collection of its members + @test collect(ah) == [ah[i] for i in keys(ah)] + @test (ah[begin], ah[end]) == (first(ah), last(collect(ah))) + @test findfirst(m -> m.name == names[end], ah) == lastindex(ah) + @test all(m.mode == 0o644 for m in ah) + + for member in ah + data = read(ah, member) + @test length(data) == member.size + # Each member is an object file of the archive's platform... + if H !== nothing + oh = only(readmeta(ah, member)) + @test oh isa H + @test isrelocatable(oh) + syms = symbol_name.(Symbols(oh)) + fn = startswith(member.name, "foo") ? "foo" : "bar" + @test mangle_symbol_name(oh, fn) in syms + # ... and reads the same as it would on its own + @test oh.header == only(readmeta(IOBuffer(data))).header + end + end + end + + @testset "thin archive" begin + ah = readmeta(open("./archives/libfoo_thin.a", "r"), ArchiveHandle) + @test isthin(ah) + @test archive_kind(ah) == :gnu + @test [m.name for m in ah] == member_names + @test all(m.offset === nothing for m in ah) + @test_throws ArgumentError read(ah, first(ah)) + @test_throws ArgumentError readmeta(ah, first(ah)) + end + + @testset "import library" begin + # The members of an import library all name the DLL they import from + ah = readmeta(open("./archives/libfoo.dll.a", "r"), ArchiveHandle) + @test archive_kind(ah) == :gnu + @test all(m.name == "foo.dll" for m in ah) + end + + @testset "kinds and names" begin + # Build an archive out of `name => data` members + function ar(members...; magic = "!\n") + io = IOBuffer() + write(io, magic) + for (name, data) in members + write(io, rpad(name, 16), rpad("0", 12), rpad("0", 6), rpad("0", 6), + rpad("644", 8), rpad(string(sizeof(data)), 10), "`\n", data) + isodd(sizeof(data)) && write(io, "\n") + end + return readmeta(IOBuffer(take!(io)), ArchiveHandle) + end + names(ah) = [m.name for m in ah] + + # An empty archive is the same in every kind + @test archive_kind(ar()) == :gnu + @test isempty(ar()) + # Only GNU (and COFF) member names are terminated by a "/"... + @test archive_kind(ar("foo.o/" => "data")) == :gnu + @test names(ar("foo.o/" => "data")) == ["foo.o"] + @test archive_kind(ar("foo.o" => "data")) == :bsd + @test names(ar("foo.o" => "data")) == ["foo.o"] + # ... so a symbol table's name is only special in its own kind, and place + @test names(ar("__.SYMDEF/" => "data")) == ["__.SYMDEF"] + @test_throws ArgumentError ar("foo.o" => "data", "__.SYMDEF" => "data") + @test_throws ArgumentError ar("foo.o/" => "data", "/" => "data") + @test_throws ArgumentError ar("/" => "data", "foo.o/" => "data", "//" => "foo.o/\n") + # GNU names must be terminated, and long names need a long name table + @test_throws ArgumentError ar("/" => "data", "foo.o" => "data") + @test_throws ArgumentError ar("/0" => "data") + @test_throws ArgumentError ar("//" => "foo.o/\n", "/7" => "data") + @test_throws ArgumentError ar("//" => "foo.o\n", "/0" => "data") + @test names(ar("//" => "foo.o/\n", "/0" => "data")) == ["foo.o"] + # Thin archives are always of the GNU kind + @test_throws ArgumentError ar("foo.o" => ""; magic = "!\n") + end + + @testset "not an archive" begin + # Archives are only opened when asked for + @test_throws MagicMismatch readmeta(open("./archives/libfoo_gnu.a", "r")) + @test_throws MagicMismatch readmeta(open("./linux64/libfoo.so", "r"), ArchiveHandle) + @test_throws MagicMismatch readmeta(IOBuffer(b"!\nfoo.o/"), ArchiveHandle) + end + + @testset "archive within a fat file" begin + # A universal archive holds one archive per architecture; open a slice + # by seeking to it. + data = read("./archives/libfoo_darwin.a") + offset = 4096 + fat = vcat(reinterpret(UInt8, hton.(UInt32[0xcafebabe, 1, 0x01000007, 3, offset, length(data), 12])), + zeros(UInt8, offset - 28), data) + mktempdir() do dir + fat_path = joinpath(dir, "libfoo_universal.a") + write(fat_path, fat) + open(fat_path, "r") do io + fh = readmeta(io, FatMachOHandle) + seek(io, only(fh.header.archs).offset) + ah = readmeta(io, ArchiveHandle) + @test [m.name for m in ah] == member_names + @test only(readmeta(ah, first(ah))) isa MachOHandle + end + end + end +end