diff --git a/src/Archive/Archive.jl b/src/Archive/Archive.jl new file mode 100644 index 0000000..716d55a --- /dev/null +++ b/src/Archive/Archive.jl @@ -0,0 +1,354 @@ +module Archive + +using ObjectFile, StructIO +import ObjectFile: readmeta, path +import Base: show, read, getindex, iterate, length, keys, firstindex, lastindex, eltype + +export ArchiveHandle, ArchiveMember, ArchiveMemberHeader, archive_kind, isthin + +# Each archive starts with one of these, followed by a sequence of members +const AR_MAGIC = b"!\n" +const AR_THIN_MAGIC = b"!\n" + +# Each member starts with an `ArchiveMemberHeader` ending in `AR_FMAG` +const AR_FMAG = b"`\n" + +# The members that index an archive rather than being a part of it, by kind of +# archive (see `archive_kind()`) +const GNU_SYMTAB = "/" +const GNU_SYMTAB64 = "/SYM64/" +const GNU_NAMETAB = "//" +const COFF_EC_SYMTAB = "//" +const BSD_SYMTAB = ("__.SYMDEF", "__.SYMDEF SORTED") +const BSD_SYMTAB64 = ("__.SYMDEF_64", "__.SYMDEF_64 SORTED") +const BSD_LONG_NAME_PREFIX = "#1/" + +""" + ArchiveMemberHeader + +The header of a member of a static archive (`ar_hdr`), made of space-padded +ASCII fields. `name` is the member's name, as encoded by the kind of archive (see +`archive_kind()`), `mode` is in octal, and the other fields are in decimal. +""" +@io struct ArchiveMemberHeader + name::NTuple{16,UInt8} + mtime::NTuple{12,UInt8} + uid::NTuple{6,UInt8} + gid::NTuple{6,UInt8} + mode::NTuple{8,UInt8} + size::NTuple{10,UInt8} + fmag::NTuple{2,UInt8} +end align_packed + +field_string(field::NTuple{N,UInt8}) where {N} = rstrip(String(collect(field)), ' ') + +# Parse an integer field of a member header, which may be blank +function parse_field(field::NTuple{N,UInt8}, base::Int = 10) where {N} + s = field_string(field) + return isempty(s) ? 0 : parse(Int64, s; base) +end + +""" + ArchiveMember + +A member of a static archive, as listed in an [`ArchiveHandle`](@ref). `offset` +is the location of the member's data relative to the start of the archive, or +`nothing` for members of a thin archive, whose data lives in the file `name` +(relative to the archive). +""" +struct ArchiveMember + name::String + offset::Union{Nothing,Int64} + size::Int64 + mtime::Int64 + uid::Int64 + gid::Int64 + mode::UInt32 +end + +""" + ArchiveHandle + +A handle on a static archive (an `ar` archive, e.g. `libfoo.a` or `foo.lib`). +Iterate over it to get the [`ArchiveMember`](@ref)s, then `readmeta(ah, member)` +to open one as an object file, or `read(ah, member)` to get its contents. + +The `ar` format comes in several kinds (see [`archive_kind()`](@ref)), which +differ in how they name members and index symbols. GNU thin archives, whose +members are stored outside of the archive, are supported too (see `isthin()`). + +Archives are not opened by `readmeta(io)`, as they are a collection of object +files, not one. Use `readmeta(io, ArchiveHandle)` instead. +""" +struct ArchiveHandle{T<:IO} + # Backing IO and start point within the IOStream of this archive + io::T + start::Int64 + + # The kind of archive, see `archive_kind()` + kind::Symbol + + # Whether this is a GNU thin archive, whose members are stored outside of it + thin::Bool + + # Members, in archive order, excluding symbol and name tables + members::Vector{ArchiveMember} + + # The path of the file this was created with, if it exists + path::String +end + +# A member as framed by its header within the archive, before its name (which +# may refer to data elsewhere, see `member_name()`) is resolved +struct RawMember + header_offset::Int64 + header::ArchiveMemberHeader + data_offset::Int64 + size::Int64 +end + +# The member's name field, without its padding +name_field(raw::RawMember) = field_string(raw.header.name) + +function malformed(raw::RawMember, msg) + throw(ArgumentError("Malformed archive member at offset $(raw.header_offset): $(msg)")) +end + +function read_raw_members(io::IO, start::Integer, thin::Bool) + raw = RawMember[] + while !eof(io) + header_offset = position(io) - start + header = unpack(io, ArchiveMemberHeader) + if collect(header.fmag) != AR_FMAG + throw(ArgumentError("Malformed archive member header at offset $(header_offset)")) + end + data_offset = position(io) - start + size = parse_field(header.size) + push!(raw, RawMember(header_offset, header, data_offset, size)) + + # Skip to the next header, which is aligned to an even offset. A thin + # archive (which is always of the GNU kind) only stores the members that + # index it within itself. + if !thin || is_index_member(:gnu, field_string(header.name)) + next_offset = data_offset + size + seek(io, start + next_offset + isodd(next_offset)) + end + end + return raw +end + +is_gnu(kind::Symbol) = kind in (:gnu, :gnu64) +is_bsd(kind::Symbol) = kind in (:bsd, :bsd64) + +# Whether `name` is that of a member which indexes an archive of the given kind +# (a symbol or long name table), rather than being a part of it +function is_index_member(kind::Symbol, name::AbstractString) + if is_bsd(kind) + return name in BSD_SYMTAB || name in BSD_SYMTAB64 + end + return name in (GNU_SYMTAB, GNU_SYMTAB64, GNU_NAMETAB, COFF_EC_SYMTAB) +end + +# Look up a name stored at `offset` in the long name table. GNU terminates names +# with "/\n", COFF with a NUL. +function lookup_long_name(raw::RawMember, kind::Symbol, table::Union{Nothing,Vector{UInt8}}) + field = name_field(raw) + offset = tryparse(Int, field[2:end]) + if table === nothing + malformed(raw, "long name $(repr(field)) without a long name table") + elseif offset === nothing || !(0 <= offset < length(table)) + malformed(raw, "long name $(repr(field)) is outside of the long name table") + end + terminator = kind == :coff ? 0x00 : UInt8('\n') + stop = findnext(==(terminator), table, offset + 1) + if stop === nothing + malformed(raw, "long name $(repr(field)) is not terminated") + end + name = String(table[offset+1:stop-1]) + if kind != :coff + endswith(name, "/") || malformed(raw, "long name $(repr(name)) is not terminated by \"/\"") + name = name[1:end-1] + end + return name +end + +""" + member_name(io, start, raw, kind, long_names) + +Resolve the name of the member `raw` of an archive (starting at `start` within +`io`) of the given `kind`, returning it along with how many bytes at the start of +the member's data it takes up: + + - BSD stores names that are long (or contain spaces) at the start of the + member's data, naming the member `#1/`. + - GNU and COFF refer to long names as `/` into the `long_names` table, + and terminate short names with a "/" (to allow for trailing spaces). The + members that index the archive have reserved names (e.g. `/` and `//`), + which are returned as-is. +""" +function member_name(io::IO, start::Integer, raw::RawMember, kind::Symbol, long_names) + field = name_field(raw) + if is_bsd(kind) + startswith(field, BSD_LONG_NAME_PREFIX) || return field, 0 + name_size = tryparse(Int, field[length(BSD_LONG_NAME_PREFIX)+1:end]) + if name_size === nothing || !(0 <= name_size <= raw.size) + malformed(raw, "invalid BSD long name $(repr(field))") + end + seek(io, start + raw.data_offset) + return rstrip(String(read(io, name_size)), '\0'), name_size + end + + if is_index_member(kind, field) + return field, 0 + elseif startswith(field, "/") + return lookup_long_name(raw, kind, long_names), 0 + elseif endswith(field, "/") + return field[1:end-1], 0 + end + malformed(raw, "member name $(repr(field)) is not terminated by \"/\"") +end + +# Determine the kind of archive from its first member(s), along with how many of +# them are symbol tables +function detect_kind(io::IO, start::Integer, raw::Vector{RawMember}) + # An empty archive is the same in every kind + isempty(raw) && return :gnu, 0 + + field = name_field(raw[1]) + if field == GNU_SYMTAB + # COFF archives start with two linker members, both named like a GNU symbol table + return length(raw) > 1 && name_field(raw[2]) == GNU_SYMTAB ? (:coff, 2) : (:gnu, 1) + elseif field == GNU_SYMTAB64 + return :gnu64, 1 + elseif startswith(field, "/") || endswith(field, "/") + # Without a symbol table, a GNU archive starts with its name table ("//") + # or a member, whose names are always terminated (or referred to) by a "/". + return :gnu, 0 + end + + # Otherwise this is a BSD archive, possibly starting with its symbol table + name = first(member_name(io, start, raw[1], :bsd, nothing)) + return name in BSD_SYMTAB64 ? (:bsd64, 1) : (:bsd, Int(name in BSD_SYMTAB)) +end + +function readmeta(io::IO, ::Type{ArchiveHandle}) + start = position(io) + magic = read(io, length(AR_MAGIC)) + thin = magic == AR_THIN_MAGIC + if magic != AR_MAGIC && !thin + throw(MagicMismatch("Magic Number $(repr(String(magic))) does not match expected archive magic number $(repr(String(AR_MAGIC)))")) + end + + raw = read_raw_members(io, start, thin) + kind, num_symtabs = detect_kind(io, start, raw) + if thin && !is_gnu(kind) + throw(ArgumentError("Malformed archive: thin archives must be of the GNU kind, not $(kind)")) + end + + # Skip over the members that index the archive, which come first: the symbol + # table(s), then (for GNU and COFF) an optional long name table, then (for + # COFF) an optional ARM64EC symbol table. + idx = 1 + num_symtabs + long_names = nothing + if !is_bsd(kind) && idx <= length(raw) && name_field(raw[idx]) == GNU_NAMETAB + seek(io, start + raw[idx].data_offset) + long_names = read(io, raw[idx].size) + idx += 1 + end + if kind == :coff && idx <= length(raw) && name_field(raw[idx]) == COFF_EC_SYMTAB + idx += 1 + end + + members = ArchiveMember[] + for r in raw[idx:end] + name, name_size = member_name(io, start, r, kind, long_names) + if is_index_member(kind, name) + malformed(r, "unexpected $(repr(name)) member in a $(kind) archive") + end + push!(members, ArchiveMember( + name, + thin ? nothing : r.data_offset + name_size, + r.size - name_size, + parse_field(r.header.mtime), + parse_field(r.header.uid), + parse_field(r.header.gid), + UInt32(parse_field(r.header.mode, 8)), + )) + end + + return ArchiveHandle(io, Int64(start), kind, thin, members, path(io)) +end + +""" + archive_kind(ah::ArchiveHandle) + +Return the kind of the archive `ah`, which determines how it names its members +and indexes their symbols: + + - `:gnu`: System V / GNU archives, with a `/` symbol table, and a `//` table of + long names (referred to as `/`) + - `:gnu64`: GNU archives with a 64-bit (`/SYM64/`) symbol table + - `:bsd`: BSD (incl. Darwin) archives, with a `__.SYMDEF` symbol table, and long + names stored before the member's data (named `#1/`) + - `:bsd64`: BSD archives with a 64-bit (`__.SYMDEF_64`) symbol table + - `:coff`: COFF (e.g. MSVC) archives, which start with two `/` linker members, + and have GNU-like names (but NUL-terminated long names) + +Archives without a symbol table are told apart by their member names, as only +GNU (and COFF) names are terminated by a `/`. +""" +archive_kind(ah::ArchiveHandle) = ah.kind + +""" + isthin(ah::ArchiveHandle) + +Returns `true` if `ah` is a GNU thin archive, whose members are not stored within +the archive, but referred to by their path relative to it. +""" +isthin(ah::ArchiveHandle) = ah.thin + +path(ah::ArchiveHandle) = ah.path + +# An archive is an indexable collection of its members +keys(ah::ArchiveHandle) = keys(ah.members) +iterate(ah::ArchiveHandle, idx=1) = iterate(ah.members, idx) +length(ah::ArchiveHandle) = length(ah.members) +firstindex(ah::ArchiveHandle) = firstindex(ah.members) +lastindex(ah::ArchiveHandle) = lastindex(ah.members) +eltype(::Type{<:ArchiveHandle}) = ArchiveMember +getindex(ah::ArchiveHandle, idx) = ah.members[idx] + +# Seek to the data of `member`, which must be stored within the archive +function seek_member(ah::ArchiveHandle, member::ArchiveMember) + if member.offset === nothing + throw(ArgumentError("Member $(member.name) of thin archive $(ah.path) is stored outside of the archive")) + end + seek(ah.io, ah.start + member.offset) +end + +""" + read(ah::ArchiveHandle, member::ArchiveMember) + +Read the contents of `member` out of the archive `ah`. +""" +function read(ah::ArchiveHandle, member::ArchiveMember) + seek_member(ah, member) + return read(ah.io, member.size) +end + +""" + readmeta(ah::ArchiveHandle, member::ArchiveMember) + +Read the object file stored as `member` of the archive `ah`, as `readmeta(io)` +would. The resulting handles share their IO with `ah`. +""" +function readmeta(ah::ArchiveHandle, member::ArchiveMember) + seek_member(ah, member) + return readmeta(ah.io) +end + +function show(io::IO, ah::ArchiveHandle) + print(io, "Archive Handle ($(archive_kind(ah)), $(length(ah)) members$(isthin(ah) ? ", thin" : ""))") +end + +end # module Archive diff --git a/src/COFF/COFFDynamicLink.jl b/src/COFF/COFFDynamicLink.jl index 6c9e129..cb6290b 100644 --- a/src/COFF/COFFDynamicLink.jl +++ b/src/COFF/COFFDynamicLink.jl @@ -34,6 +34,11 @@ function find_section_for_rva(oh::H, rva) where {H <: COFFHandle} end function COFFDynamicLinks(oh::H) where {H <: COFFHandle} + # Object files are not loaded, and so have no import table + if oh.opt_header === nothing + return COFFDynamicLinks(oh, COFFDynamicLink{H}[]) + end + # Start by finding the virtual address of the import table import_table_rva = oh.opt_header.directories.ImportTable.VirtualAddress diff --git a/src/COFF/COFFHandle.jl b/src/COFF/COFFHandle.jl index f4ff1d8..b5d1f50 100644 --- a/src/COFF/COFFHandle.jl +++ b/src/COFF/COFFHandle.jl @@ -17,8 +17,9 @@ struct COFFHandle{T<:IO} <: ObjectHandle # The location of the header (because of MZ confusion, we must store this) header_offset::UInt32 - # The "Optional" header, which isn't actually optional - opt_header::COFFOptionalHeader + # The "Optional" header, which isn't actually optional for images, but is + # absent from object files + opt_header::Union{Nothing,COFFOptionalHeader} # The path of the file this was created with path::String @@ -35,22 +36,32 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle} # Check to see if this is an 'MZ' file; if it is, we have to jump forward # into the file a bit to get to the PE header - magic = [read(io, UInt8) for idx in 1:2] - if all(magic .== MZ_magic) + magic = read(io, 2) + if magic == MZ_magic # Skip ahead to the PE header offset skip(io, 58) header_offset = read(io, UInt32) seek(io, start + header_offset) else - # If it's not, assume the PE header starts at the beginning of the file - skip(io, -2) + # If it's not, the PE header may start at the beginning of the file + seek(io, start) end - magic = [read(io, UInt8) for idx in 1:4] - if any(magic .!= PE_magic) + magic = read(io, 4) + if magic != PE_magic + # An object file has no PE header (nor an MZ stub), it starts directly + # with its COFF header. + if magic[1:min(2, end)] != MZ_magic + seek(io, start) + header = read_object_header(io) + if header !== nothing + return [COFFHandle(io, Int64(start), header, UInt32(0), nothing, path(io))] + end + end + msg = """ Magic Number 0x$(join(string.(magic, base=16, pad=2),"")) does not match expected PE - magic number 0x$(join(string.(PE_magic, base=16, pad=2),"")) + magic number 0x$(join(string.(PE_magic, base=16, pad=2),"")), nor is this a COFF object """ throw(MagicMismatch(replace(strip(msg), "\n" => " "))) end @@ -66,6 +77,40 @@ function readmeta(io::IO, ::Type{H}) where {H <: COFFHandle} return [COFFHandle(io, Int64(start), header, header_offset, opt_header, path(io))] end +""" + read_object_header(io::IO) + +Read the `COFFHeader` of an object file (e.g. `foo.obj`) at the current position +of `io`, returning `nothing` if it does not look like one. Object files have no +magic number beyond the `Machine` field of their header, so we check that the +rest of the header is consistent with an object file of that size. +""" +function read_object_header(io::IO) + start = position(io) + header = unpack(io, COFFHeader) + # Find out how much room there is for this object, at most + seekend(io) + available = position(io) - start + seek(io, start) + + if !haskey(IMAGE_FILE_MACHINE, header.Machine) || header.Machine == IMAGE_FILE_MACHINE_UNKNOWN + return nothing + end + # Only images have an optional header, or are executables/DLLs + if header.SizeOfOptionalHeader != 0 || + (header.Characteristics & (IMAGE_FILE_EXECUTABLE_IMAGE | IMAGE_FILE_DLL)) != 0 + return nothing + end + # The section and symbol tables must fit within the object + if sizeof(COFFHeader) + header.NumberOfSections * packed_sizeof(COFFSection{COFFHandle}) > available + return nothing + end + if header.PointerToSymbolTable != 0 && + header.PointerToSymbolTable + header.NumberOfSymbols * packed_sizeof(COFFSymtabEntry{COFFHandle}) > available + return nothing + end + return header +end ## IOStream-like operations: startaddr(oh::COFFHandle) = oh.start @@ -82,11 +127,11 @@ islibrary(oh::COFFHandle) = islibrary(header(oh)) isdynamic(oh::COFFHandle) = !isempty(findall(Sections(oh), [".idata"])) mangle_section_name(oh::COFFHandle, name::AbstractString) = string(".", name) function mangle_symbol_name(oh::COFFHandle, name::AbstractString) - # sob - if is64bit(oh) - return name - else + # sob; only 32-bit x86 prefixes C symbols with an underscore + if header(oh).Machine == IMAGE_FILE_MACHINE_I386 return string("_", name) + else + return name end end format_string(::Type{H}) where {H <: COFFHandle} = "COFF" diff --git a/src/COFF/COFFHeader.jl b/src/COFF/COFFHeader.jl index 577d1a2..2d7d3d8 100644 --- a/src/COFF/COFFHeader.jl +++ b/src/COFF/COFFHeader.jl @@ -16,6 +16,8 @@ function coff_header_is64bit(h::COFFHeader) wide_machines = [ IMAGE_FILE_MACHINE_AMD64, IMAGE_FILE_MACHINE_ARM64, + IMAGE_FILE_MACHINE_ARM64EC, + IMAGE_FILE_MACHINE_ARM64X, IMAGE_FILE_MACHINE_IA64, ] return h.Machine in wide_machines diff --git a/src/COFF/COFFSymbol.jl b/src/COFF/COFFSymbol.jl index 2d7d938..da7b2e8 100644 --- a/src/COFF/COFFSymbol.jl +++ b/src/COFF/COFFSymbol.jl @@ -5,61 +5,60 @@ export COFFSymbols, COFFSymtabEntry, COFFSymbolRef COFF symbol table, contains the list of symbols defined within the object file. -Note that because COFF Symbols are variable-length, we store a table of offsets -at which the (non-auxilliary) symbols can be found. +Note that because COFF Symbols are followed by a variable number of auxiliary +records within the symbol table, we store the (1-based) symbol table index at +which each (non-auxiliary) symbol can be found. """ struct COFFSymbols{H<:COFFHandle} <: Symbols{H} handle::H - symbol_offsets::Vector{UInt64} + symbol_indices::Vector{UInt32} end """ - scan_symbol_offsets(oh::COFFHandle) + scan_symbol_indices(oh::COFFHandle) -Find the offsets for each Symbol within a COFFHandle, skipping over auxilliary -symbols as necessary. +Find the symbol table index of each Symbol within a COFFHandle, skipping over +auxiliary records as necessary. """ -function scan_symbol_offsets(oh::H) where {H <: COFFHandle} - # num_syms represents the number of total symbols, but we're only +function scan_symbol_indices(oh::H) where {H <: COFFHandle} + # num_syms represents the number of total records, but we're only # interested in the non-auxiliary symbols num_syms = num_symbols(header(oh)) - offsets = UInt64[] + indices = UInt32[] - curr_idx = 1 idx = 1 - while idx < num_syms - # Load in the next symbol and store its offset + while idx <= num_syms + # Load in the next symbol and store its index seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(idx-1)) + sym = unpack(oh, symtab_entry_type(oh)) + push!(indices, idx) - sym = unpack(oh, COFFSymtabEntry{H}) - push!(offsets, idx) - - # Move our offset forward by 1 (because we just read in a SymtabEntry) - # and again as many times as needed to skip over the auxilliary symbols + # Move our index forward by 1 (because we just read in a SymtabEntry) + # and again as many times as needed to skip over the auxiliary records idx += 1 + sym.NumberOfAuxSymbols - curr_idx += 1 end - return offsets + return indices end function Symbols(oh::H) where {H <: COFFHandle} - symbol_offsets = scan_symbol_offsets(oh) - return COFFSymbols(oh, symbol_offsets) + return COFFSymbols(oh, scan_symbol_indices(oh)) end handle(syms::COFFSymbols) = syms.handle -# function next(syms::COFFSymbols, idx) -# # We skip over auxiliary symbols here -# next_idx = idx + deref(syms[idx]).NumberOfAuxSymbols + 1 -# return (syms[idx], next_idx) -# end -lastindex(syms::COFFSymbols) = num_symbols(header(handle(syms))) - -# We override `iteratorsize` so that list comprehensions and collect() calls on -# our Symbols don't have a bunch of #undef entries at the end of the array -# because it tries to pre-allocate an array of the proper size. -# iteratorsize(::Type{H}) where {H <: COFFSymbols} = SizeUnknown() +lastindex(syms::COFFSymbols) = length(syms.symbol_indices) + +# Index into the symbols, skipping over auxiliary records. The resulting +# `symbol_number()` is the symbol's index within the symbol table. +function getindex(syms::COFFSymbols, idx) + if !(0 < idx <= length(syms)) + throw(BoundsError(syms, idx)) + end + oh = handle(syms) + table_idx = syms.symbol_indices[idx] + seek(oh, symtab_entry_offset(oh) + symtab_entry_size(oh)*(table_idx-1)) + return SymbolRef(syms, unpack(oh, symtab_entry_type(oh)), table_idx) +end @io struct COFFSymtabEntry{H <: COFFHandle} <: SymtabEntry{H} Name::fixed_string{UInt64} diff --git a/src/COFF/constants.jl b/src/COFF/constants.jl index d1569bd..c9be3e0 100644 --- a/src/COFF/constants.jl +++ b/src/COFF/constants.jl @@ -6,6 +6,8 @@ const IMAGE_FILE_MACHINE_ARM = 0x1c0 # ARM little endian const IMAGE_FILE_MACHINE_ARMNT = 0x1c4 # ARMv7 (or higher) Thumb mode only const IMAGE_FILE_MACHINE_ARM64 = 0xaa64 # ARMv8 in 64-bit mode + const IMAGE_FILE_MACHINE_ARM64EC = 0xa641 # ARMv8 in 64-bit mode, with the x64 ABI + const IMAGE_FILE_MACHINE_ARM64X = 0xa64e # ARM64 and ARM64EC code together const IMAGE_FILE_MACHINE_EBC = 0xebc # EFI byte code const IMAGE_FILE_MACHINE_I386 = 0x14c # Intel 386 or later processors and compatible processors const IMAGE_FILE_MACHINE_IA64 = 0x200 # Intel Itanium processor family @@ -31,7 +33,7 @@ function coff_machine_to_arch(machine::UInt16) return "x86_64" elseif machine ∈ (IMAGE_FILE_MACHINE_ARM, IMAGE_FILE_MACHINE_ARMNT, IMAGE_FILE_MACHINE_THUMB) return "armv7l" - elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64,) + elseif machine ∈ (IMAGE_FILE_MACHINE_ARM64, IMAGE_FILE_MACHINE_ARM64EC, IMAGE_FILE_MACHINE_ARM64X) return "aarch64" elseif machine ∈ (IMAGE_FILE_MACHINE_POWERPC,) return "ppc64le" diff --git a/src/ObjectFile.jl b/src/ObjectFile.jl index 6ef1bb8..254234a 100644 --- a/src/ObjectFile.jl +++ b/src/ObjectFile.jl @@ -21,6 +21,10 @@ include("MachO/MachO.jl") include("COFF/COFF.jl") @reexport using .COFF +# Include static archives (`ar`), which hold object files of the above formats +include("Archive/Archive.jl") +@reexport using .Archive + function __init__() global ObjTypes @@ -28,6 +32,8 @@ function __init__() push!(ObjTypes, MachOHandle) push!(ObjTypes, FatMachOHandle) push!(ObjTypes, COFFHandle) + # `ArchiveHandle` is deliberately not registered: `readmeta(io)` returns + # object files, and an archive is a collection of them. end end #module ObjectFile diff --git a/test/archives/build.sh b/test/archives/build.sh new file mode 100755 index 0000000..32bdbe8 --- /dev/null +++ b/test/archives/build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Regenerates the static archive fixtures in this directory. Requires `clang`, +# `llvm-ar`, `llvm-lib` and `llvm-dlltool`. The object file names are chosen to +# exercise both short member names and those that need a long name table. +set -euo pipefail +cd "$(dirname "$0")" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +echo 'int foo(int x) { return x + 1; }' > "${tmp}/foo.c" +echo 'int foo(int x); int bar(int x) { return foo(x) * 2; }' > "${tmp}/bar_with_a_long_name.c" + +compile() { + for src in foo bar_with_a_long_name; do + clang --target="$@" -O2 -c -o "${tmp}/${src}.o" "${tmp}/${src}.c" + done +} +# `archive [ops]`, where `ops` defaults to `rcsD` (with a symbol table) +archive() { + local format="$1" out="$2" ops="${3:-rcsD}" + rm -f "${out}" + (cd "${tmp}" && llvm-ar --format="${format}" "${ops}" "${OLDPWD}/${out}" foo.o bar_with_a_long_name.o) +} + +compile x86_64-linux-gnu +archive gnu libfoo_gnu.a +archive gnu libfoo_gnu_nosymtab.a rcSD +# (`SYM64_THRESHOLD` forces a 64-bit symbol table) +SYM64_THRESHOLD=0 archive gnu libfoo_gnu64.a +# Some archivers record member paths relative to the archive, so build those beside them +(cd "${tmp}" && llvm-ar --format=gnu --thin rcsD libfoo_thin.a foo.o bar_with_a_long_name.o) +mv "${tmp}/libfoo_thin.a" libfoo_thin.a + +compile x86_64-apple-macos +archive darwin libfoo_darwin.a +SYM64_THRESHOLD=0 archive darwin libfoo_darwin64.a +archive bsd libfoo_bsd_nosymtab.a rcSD + +# (`-mno-incremental-linker-compatible` keeps timestamps out of COFF objects) +compile x86_64-pc-windows-msvc -mno-incremental-linker-compatible +(cd "${tmp}" && llvm-lib /out:foo.lib foo.o bar_with_a_long_name.o) +mv "${tmp}/foo.lib" foo.lib + +# An import library: an archive too, but of short import objects +printf 'LIBRARY foo.dll\nEXPORTS\nfoo\nbar\n' > "${tmp}/foo.def" +rm -f libfoo.dll.a +llvm-dlltool -m i386:x86-64 -d "${tmp}/foo.def" -l libfoo.dll.a diff --git a/test/archives/foo.lib b/test/archives/foo.lib new file mode 100644 index 0000000..269733b Binary files /dev/null and b/test/archives/foo.lib differ diff --git a/test/archives/libfoo.dll.a b/test/archives/libfoo.dll.a new file mode 100644 index 0000000..e65e858 Binary files /dev/null and b/test/archives/libfoo.dll.a differ diff --git a/test/archives/libfoo_bsd_nosymtab.a b/test/archives/libfoo_bsd_nosymtab.a new file mode 100644 index 0000000..c385379 Binary files /dev/null and b/test/archives/libfoo_bsd_nosymtab.a differ diff --git a/test/archives/libfoo_darwin.a b/test/archives/libfoo_darwin.a new file mode 100644 index 0000000..a00ed9a Binary files /dev/null and b/test/archives/libfoo_darwin.a differ diff --git a/test/archives/libfoo_darwin64.a b/test/archives/libfoo_darwin64.a new file mode 100644 index 0000000..d3a7b3f Binary files /dev/null and b/test/archives/libfoo_darwin64.a differ diff --git a/test/archives/libfoo_gnu.a b/test/archives/libfoo_gnu.a new file mode 100644 index 0000000..257939e Binary files /dev/null and b/test/archives/libfoo_gnu.a differ diff --git a/test/archives/libfoo_gnu64.a b/test/archives/libfoo_gnu64.a new file mode 100644 index 0000000..1e473d0 Binary files /dev/null and b/test/archives/libfoo_gnu64.a differ diff --git a/test/archives/libfoo_gnu_nosymtab.a b/test/archives/libfoo_gnu_nosymtab.a new file mode 100644 index 0000000..2b66a76 Binary files /dev/null and b/test/archives/libfoo_gnu_nosymtab.a differ diff --git a/test/archives/libfoo_thin.a b/test/archives/libfoo_thin.a new file mode 100644 index 0000000..b0c0dae Binary files /dev/null and b/test/archives/libfoo_thin.a differ diff --git a/test/coff/build.sh b/test/coff/build.sh new file mode 100755 index 0000000..fea0291 --- /dev/null +++ b/test/coff/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Regenerates the COFF object fixtures in this directory. Requires `clang`. +set -euo pipefail +cd "$(dirname "$0")" +tmp="$(mktemp -d)" +trap 'rm -rf "${tmp}"' EXIT + +echo 'int foo(int x) { return x + 1; }' > "${tmp}/foo.c" + +# (`-mno-incremental-linker-compatible` keeps timestamps out of COFF objects) +for arch in x86_64 i686 aarch64 thumbv7; do + clang --target="${arch}-pc-windows-msvc" -O2 -mno-incremental-linker-compatible \ + -c -o "foo_${arch}.obj" "${tmp}/foo.c" +done diff --git a/test/coff/foo_aarch64.obj b/test/coff/foo_aarch64.obj new file mode 100644 index 0000000..b32bf13 Binary files /dev/null and b/test/coff/foo_aarch64.obj differ diff --git a/test/coff/foo_i686.obj b/test/coff/foo_i686.obj new file mode 100644 index 0000000..763cc2a Binary files /dev/null and b/test/coff/foo_i686.obj differ diff --git a/test/coff/foo_thumbv7.obj b/test/coff/foo_thumbv7.obj new file mode 100644 index 0000000..3f45172 Binary files /dev/null and b/test/coff/foo_thumbv7.obj differ diff --git a/test/coff/foo_x86_64.obj b/test/coff/foo_x86_64.obj new file mode 100644 index 0000000..cdc1e77 Binary files /dev/null and b/test/coff/foo_x86_64.obj differ diff --git a/test/runtests.jl b/test/runtests.jl index c5b1146..654f4a3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -139,6 +139,15 @@ function test_libfoo_and_fooifier(fooifier_path, libfoo_path) @test !islocal(syms_exe[main_idx_exe]) @test !islocal(syms_lib[foo_idx_lib]) + # COFF symbols are followed by auxiliary records, which are skipped + if isa(oh_exe, COFFHandle) + for (oh, syms) in ((oh_exe, syms_exe), (oh_lib, syms_lib)) + naux = [ObjectFile.deref(sym).NumberOfAuxSymbols for sym in syms] + @test length(syms) + sum(naux) == header(oh).NumberOfSymbols + @test ObjectFile.symbol_number(syms[end]) + naux[end] == header(oh).NumberOfSymbols + end + end + # Global detection doesn't seem to be working on OSX... if !isa(oh_exe, MachOHandle) @test isglobal(syms_exe[foo_idx_exe]) @@ -337,3 +346,168 @@ using Mmap find_dep_libs(joinpath("./libjulias", file)) end end + +# Fixtures are generated by `./archives/build.sh` +@testset "Archives" begin + member_names = ["foo.o", "bar_with_a_long_name.o"] + + @testset "$(file)" for (file, kind, H, names) in ( + ("libfoo_gnu.a", :gnu, ELFHandle, member_names), + ("libfoo_gnu_nosymtab.a", :gnu, ELFHandle, member_names), + ("libfoo_gnu64.a", :gnu64, ELFHandle, member_names), + ("libfoo_darwin.a", :bsd, MachOHandle, member_names), + ("libfoo_darwin64.a", :bsd64, MachOHandle, member_names), + ("libfoo_bsd_nosymtab.a", :bsd, MachOHandle, member_names), + # `llvm-lib` orders members by name + ("foo.lib", :coff, COFFHandle, reverse(member_names)), + ) + ah = readmeta(open(joinpath("./archives", file), "r"), ArchiveHandle) + @test archive_kind(ah) == kind + @test !isthin(ah) + @test length(ah) == 2 + @test eltype(ah) == ArchiveMember + # Symbol and name tables are not listed as members + @test [m.name for m in ah] == names + # An archive is an indexable collection of its members + @test collect(ah) == [ah[i] for i in keys(ah)] + @test (ah[begin], ah[end]) == (first(ah), last(collect(ah))) + @test findfirst(m -> m.name == names[end], ah) == lastindex(ah) + @test all(m.mode == 0o644 for m in ah) + + for member in ah + data = read(ah, member) + @test length(data) == member.size + # Each member is an object file of the archive's platform... + if H !== nothing + oh = only(readmeta(ah, member)) + @test oh isa H + @test isrelocatable(oh) + syms = symbol_name.(Symbols(oh)) + fn = startswith(member.name, "foo") ? "foo" : "bar" + @test mangle_symbol_name(oh, fn) in syms + # `bar()` calls `foo()`, which another member defines + foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo")) + @test isundef(foo_sym) == (fn == "bar") + # ... and reads the same as it would on its own + @test oh.header == only(readmeta(IOBuffer(data))).header + end + end + end + + @testset "thin archive" begin + ah = readmeta(open("./archives/libfoo_thin.a", "r"), ArchiveHandle) + @test isthin(ah) + @test archive_kind(ah) == :gnu + @test [m.name for m in ah] == member_names + @test all(m.offset === nothing for m in ah) + @test_throws ArgumentError read(ah, first(ah)) + @test_throws ArgumentError readmeta(ah, first(ah)) + end + + @testset "import library" begin + # The members of an import library all name the DLL they import from + ah = readmeta(open("./archives/libfoo.dll.a", "r"), ArchiveHandle) + @test archive_kind(ah) == :gnu + @test all(m.name == "foo.dll" for m in ah) + end + + @testset "kinds and names" begin + # Build an archive out of `name => data` members + function ar(members...; magic = "!\n") + io = IOBuffer() + write(io, magic) + for (name, data) in members + write(io, rpad(name, 16), rpad("0", 12), rpad("0", 6), rpad("0", 6), + rpad("644", 8), rpad(string(sizeof(data)), 10), "`\n", data) + isodd(sizeof(data)) && write(io, "\n") + end + return readmeta(IOBuffer(take!(io)), ArchiveHandle) + end + names(ah) = [m.name for m in ah] + + # An empty archive is the same in every kind + @test archive_kind(ar()) == :gnu + @test isempty(ar()) + # Only GNU (and COFF) member names are terminated by a "/"... + @test archive_kind(ar("foo.o/" => "data")) == :gnu + @test names(ar("foo.o/" => "data")) == ["foo.o"] + @test archive_kind(ar("foo.o" => "data")) == :bsd + @test names(ar("foo.o" => "data")) == ["foo.o"] + # ... so a symbol table's name is only special in its own kind, and place + @test names(ar("__.SYMDEF/" => "data")) == ["__.SYMDEF"] + @test_throws ArgumentError ar("foo.o" => "data", "__.SYMDEF" => "data") + @test_throws ArgumentError ar("foo.o/" => "data", "/" => "data") + @test_throws ArgumentError ar("/" => "data", "foo.o/" => "data", "//" => "foo.o/\n") + # GNU names must be terminated, and long names need a long name table + @test_throws ArgumentError ar("/" => "data", "foo.o" => "data") + @test_throws ArgumentError ar("/0" => "data") + @test_throws ArgumentError ar("//" => "foo.o/\n", "/7" => "data") + @test_throws ArgumentError ar("//" => "foo.o\n", "/0" => "data") + @test names(ar("//" => "foo.o/\n", "/0" => "data")) == ["foo.o"] + # Thin archives are always of the GNU kind + @test_throws ArgumentError ar("foo.o" => ""; magic = "!\n") + end + + @testset "not an archive" begin + # Archives are only opened when asked for + @test_throws MagicMismatch readmeta(open("./archives/libfoo_gnu.a", "r")) + @test_throws MagicMismatch readmeta(open("./linux64/libfoo.so", "r"), ArchiveHandle) + @test_throws MagicMismatch readmeta(IOBuffer(b"!\nfoo.o/"), ArchiveHandle) + end + + @testset "archive within a fat file" begin + # A universal archive holds one archive per architecture; open a slice + # by seeking to it. + data = read("./archives/libfoo_darwin.a") + offset = 4096 + fat = vcat(reinterpret(UInt8, hton.(UInt32[0xcafebabe, 1, 0x01000007, 3, offset, length(data), 12])), + zeros(UInt8, offset - 28), data) + mktempdir() do dir + fat_path = joinpath(dir, "libfoo_universal.a") + write(fat_path, fat) + open(fat_path, "r") do io + fh = readmeta(io, FatMachOHandle) + seek(io, only(fh.header.archs).offset) + ah = readmeta(io, ArchiveHandle) + @test [m.name for m in ah] == member_names + @test only(readmeta(ah, first(ah))) isa MachOHandle + end + end + end +end + +# Fixtures are generated by `./coff/build.sh` +@testset "COFF objects" begin + @testset "$(arch)" for (arch, platform) in ( + ("x86_64", Platform("x86_64", "windows")), + ("i686", Platform("i686", "windows")), + ("aarch64", Platform("aarch64", "windows")), + ("thumbv7", Platform("armv7l", "windows")), + ) + oh = only(readmeta(open("./coff/foo_$(arch).obj", "r"))) + @test oh isa COFFHandle + @test platforms_match(Platform(oh), platform) + @test is64bit(oh) == (arch in ("x86_64", "aarch64")) + @test isrelocatable(oh) + @test !isexecutable(oh) + @test !islibrary(oh) + @test isempty(DynamicLinks(oh)) + @test ".text" in section_name.(Sections(oh)) + foo_sym = only(sym for sym in Symbols(oh) if symbol_name(sym) == mangle_symbol_name(oh, "foo")) + @test isglobal(foo_sym) + @test !isundef(foo_sym) + end + + @testset "not an object" begin + not_obj(data) = readmeta(IOBuffer(data), COFFHandle) + @test_throws MagicMismatch not_obj(b"hello, world, this is not an object file") + # An x86_64 `Machine`, but with an optional header + @test_throws MagicMismatch not_obj(vcat(b"d\x86", zeros(UInt8, 14), [0x08, 0x00], zeros(UInt8, 2))) + # ... or with more sections than fit in the file + @test_throws MagicMismatch not_obj(vcat(b"d\x86", [0xff, 0x00], zeros(UInt8, 16))) + # An unknown `Machine` + @test_throws MagicMismatch not_obj(vcat(b"zz", zeros(UInt8, 18))) + end +end