diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1aeec22 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Stored with LF; checked out with the platform's line endings +* text=auto diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b61599e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +jobs: + # Library tests (tests/), including the compile-time leftovers matrix + tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - run: cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release + - run: cmake --build build --config Release --parallel + - run: ctest --test-dir build -C Release --output-on-failure + diff --git a/.gitignore b/.gitignore index dbe9c82..9baa48d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.vscode/ \ No newline at end of file +.vscode/ +build*/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..c72d9e3 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,9 @@ +# Header-only: for add_subdirectory / FetchContent users. Links as PatternScanner::PatternScanner. +# Tests are a separate project in tests/. +cmake_minimum_required(VERSION 3.16) +project(PatternScanner CXX) + +add_library(PatternScanner INTERFACE) +add_library(PatternScanner::PatternScanner ALIAS PatternScanner) +target_include_directories(PatternScanner INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_compile_features(PatternScanner INTERFACE cxx_std_17) diff --git a/CompileTimePattern.hpp b/CompileTimePattern.hpp deleted file mode 100644 index 14af549..0000000 --- a/CompileTimePattern.hpp +++ /dev/null @@ -1,137 +0,0 @@ -#pragma once -#include "Pattern.hpp" -#include - -namespace patterns { - - namespace detail { - constexpr size_t pattern_length(const char* s, size_t nstr, bool pad = true) { - size_t res = 0; -#ifdef __arm64__ - bool align = false; -#endif - for (auto i = 0; i < nstr - 1; i += 2) { - auto c = s[i]; - if (c == 'X' || c == 'x') { - while (s[i + 1] != ' ') - ++i; - continue; - } - else if (c == '/') { -#ifdef __arm64__ - while (i < nstr - 1 && (c = s[++i])) { - if (c == 'a') { - align = true; - break; - } - } -#endif - break; - } - else if (c != ' ') - ++res; - else --i; - } -#ifdef __arm64__ - while (pad && align && res % sizeof(uint32_t)) - ++res; -#else - while (pad && res % sizeof(void*)) - ++res; -#endif - return res; - } - - template - constexpr size_t pattern_length(const char(&s)[nstr], bool pad = true) { - return pattern_length(s, nstr, pad); - } - -#if __cplusplus > 201703L - template - struct const_string { - char data[nstr]{}; - static constexpr size_t length = nstr - 1; - static constexpr size_t size = nstr; - - constexpr const_string(const char(&s)[nstr]) { - std::copy_n(s, length, data); - } - }; -#endif - } - - template - class CompileTimePattern : public Pattern { - std::array m_pattern; - std::array m_mask; - public: - constexpr CompileTimePattern(const char* p) : - m_pattern{}, m_mask{} - { - length_ = narr; - auto n = 0; - for (auto i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (p[i] == '?') { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern[n] = value(ptr); - m_mask[n] = 0xFF; - ++n; - } - else --i; - } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#endif - } - virtual const uint8_t* pattern() const override { - return m_pattern.data(); - } - virtual const uint8_t* mask() const override { - return m_mask.data(); - } - }; -} - -#if __cplusplus > 201703L -template -constexpr auto operator"" _ctpattern() { - return patterns::CompileTimePattern(str.data); -} -#else -#ifndef COMPILETIME_PATTERN -#define COMPILETIME_PATTERN(x) patterns::CompileTimePattern(x) -#endif -#endif diff --git a/Pattern.hpp b/Pattern.hpp deleted file mode 100644 index 7be9ad2..0000000 --- a/Pattern.hpp +++ /dev/null @@ -1,444 +0,0 @@ -#pragma once -#include -#include - -#ifndef _MSVC_VER -#define __forceinline inline __attribute__((always_inline)) -#endif - -namespace patterns { - namespace detail { - constexpr bool __forceinline is_digit(char c) { - return c <= '9' && c >= '0'; - } - constexpr bool __forceinline is_hex_digit(char c) { - return is_digit(c) || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); - } - constexpr __forceinline char get_bits(char c) { - return is_digit(c) ? (c - '0') : ((c & (~0x20)) - 'A' + 0xA); - } - template - constexpr T stoi_impl(const char* str, T value = 0, bool negative = false, bool hex = false) { - if (*str == '\0') return negative ? -value : value; - if (hex) { - if (is_hex_digit(*str)) - return stoi_impl(str + 1, get_bits(*str) + value * 16, negative, hex); - } else { - if (is_digit(*str)) - return stoi_impl(str + 1, (*str - '0') + value * 10, negative, hex); - } - return negative ? -value : value; - } - template - constexpr T stoi(const char* str) { - if (*str == '-') { - ++str; - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, true, true); - return stoi_impl(str, 0, true); - } - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, false, true); - return stoi_impl(str); - } -#ifdef __arm64__ - template - T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { - constexpr size_t int_width = sizeof(int32_t) * 8; - return (static_cast(insn) << (int_width - (offset + width))) >> (int_width - width); - } - - bool __forceinline decode_masked_match(uint32_t insn, uint32_t mask, uint32_t pattern) { - return (insn & mask) == pattern; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/NOP--No-Operation- - bool a64_decode_nop(uint32_t insn) { - if (insn == 0b1101'0101'0000'0011'0010'0000'0001'1111) - return true; - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/B--Branch- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/BL--Branch-with-Link- - bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { - // x001 01?? ???? ???? ???? ???? ???? ???? - // 1 - BL - // 0 - B - if (decode_masked_match(insn, 0b0111'1100'0000'0000'0000'0000'0000'0000, 0b0001'0100'0000'0000'0000'0000'0000'0000)) { - if (is_bl) *is_bl = (insn >> 31) & 0x1; - *offset = extract_bitfield(insn, 26, 0) << 2; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADR--Form-PC-relative-address- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- - bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { - // rd - destination register - // x??1 0000 ???? ???? ???? ???? ???? ???? - // 1 - ADRP - // 0 - ADR - // bits 30 & 29 - immlo - // bits 23 -> 5 - immhi - // bits 4 -> 0 - rd - if (decode_masked_match(insn, 0b0001'1111'0000'0000'0000'0000'0000'0000, 0b0001'0000'0000'0000'0000'0000'0000'0000)) { - if (rd) *rd = insn & 0x1f; - uint32_t immlo = (insn >> 29) & 0x3; - int32_t immhi = extract_bitfield(insn, 19, 5) << 2; - auto adrp = (insn >> 31) & 0x1; - if (is_adrp) *is_adrp = adrp; - *offset = (adrp ? (immhi | immlo) * 4096 : (immhi | immlo)); - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- - bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose base register to be transfered - // Post-index & Pre-index masks - // 0111 1000 010? ???? ???? x1?? ???? ???? - // 1 - pre-index - // 0 - post-index - if (decode_masked_match(insn, 0b1111'1111'1110'0000'0000'0100'0000'0000, 0b0111'1000'0100'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 0111 1001 01?? ???? ???? ???? ???? ???? - if (decode_masked_match(insn, 0b1111'1111'1100'0000'0000'0000'0000'0000, 0b0111'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- - bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 010? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1111'1110'0000'0000'0100'0000'0000, 0b1011'1000'0100'0000'0000'0100'0000'0000)) { - // Have not tested this yet - *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 1x11 1001 01?? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1100'0000'0000'0000'0000'0000, 0b1011'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/STR--immediate---Store-Register--immediate-- - bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 000? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0100'0000'0000, 0b1011'1000'0000'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 1x11 1000 000? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0000'0000'0000, 0b1011'1001'0000'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- - bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { - // sf - 1 for 64 bit, 0 for 32 bit - // rd - destination register - // x101 0010 1xx? ???? ???? ???? ???? ???? - // 1 = 64 bit 0x - shift ammount - // 0 = 32 bit - if (decode_masked_match(insn, 0b0111'1111'1000'0000'0000'0000'0000'0000, 0b0101'0010'1000'0000'0000'0000'0000'0000)) { - auto hw = (insn >> 21) & 3; - *offset = hw ? extract_bitfield(insn, 16, 5) << hw : extract_bitfield(insn, 16, 5); - if (sf) *sf = (insn >> 31) & 0x1; - if (rd) *rd = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADD--immediate---Add--immediate-- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/SUB--immediate---Subtract--immediate-- - bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { - // rd - destination register - // rn - source register - // sf - 1 for 64 bit, 0 for 32 bit - // ?x01 0001 0??? ???? ???? ???? ???? ???? - // 1 = subtraction - // 0 = addition - if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0001'0000'0000'0000'0000'0000'0000)) { - if (sf) *sf = (insn >> 31) & 0x1; - if (rd) *rd = insn & 0x1f; - if (rn) *rn = (insn >> 5) & 0x1f; - if (is_sub) *is_sub = (insn >> 30) & 0x1; - uint32_t imm12 = extract_bitfield(insn, 12, 10); - // sh = 1 is LSL#12 - auto sh = (insn >> 22) & 0x1; - *offset = (sh ? (imm12 << 12) : imm12); - return true; - } - return false; - } -#endif - } - - class Pattern { - protected: - uint32_t length_ = 0; - uint32_t offset_ = 0; - bool deref_ = false; -#ifndef __arm64__ - // Remaining bytes from the offset when used in dereferencing - uint32_t insn_len_ = 0; - // Since all arm64 instructions are 32 bit and encoded, just doing deref instead of both deref and relative - // Defaulting 4 byte relative address reading - uint32_t size_ = 4; - bool rel_ = false; -#endif - bool align_ = false; -#ifdef __arm64__ - // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level - static constexpr size_t align_size_ = 4; -#else - static constexpr size_t align_size_ = sizeof(void*); -#endif - public: - Pattern() = default; - ~Pattern() = default; - const uint32_t __forceinline length() const { - return length_; - } - const uint32_t __forceinline offset() const { - return offset_; - } - const bool __forceinline deref() const { - return deref_; - } -#ifndef __arm64__ - const bool __forceinline relative() const { - return rel_; - } -#endif - const bool __forceinline aligned() const { - return align_; - } - virtual const uint8_t* mask() const = 0; - virtual const uint8_t* pattern() const = 0; - template - T find(const uint8_t* bytes, size_t size) const { - return reinterpret_cast(find(bytes, size)); - } - virtual void* find(const uint8_t* bytes, size_t size) const { - void* result = nullptr; - const auto pattern = const_cast(this->pattern()); - const auto mask = const_cast(this->mask()); - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; -#ifdef __arm64__ - // Doing byte by byte match due to arm being encoded instructions, unless specified to align scan - if (align_) { - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } - } else { - for (auto j = 0U; j < length_; ++j) { - if (mask[j] == 0xFF && pattern[j] != i[j]) { - found = false; - break; - } - } - } -#else - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } -#endif - if (found) - return get_result(i, end); - } - return result; - } - virtual uint8_t operator[](size_t idx) const { - return pattern()[idx]; - } - protected: - // Credits to EJT for the helper functions here! - constexpr __forceinline uint8_t value(const char* c) const { - return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); - } - constexpr void handle_options(const char* ptr) { - while (*ptr) { -#ifdef __arm64__ - if (*ptr == 'd') - deref_ = true; -#else - if (*ptr == 'd') { - if (rel_) throw std::logic_error("Cannot use relative and deref together!"); - deref_ = true; - } - else if (*ptr == 'r') { - if (deref_) throw std::logic_error("Cannot use relative and deref together!"); - rel_ = true; - } -#endif - else if (*ptr == 'a') - align_ = true; -#ifndef __arm64__ - // Check the next character to see what size we're reading at this relative address - else if (*ptr > '0' && (sizeof(void*) == 0x8 ? *ptr < '9' : *ptr < '5')) { - size_ = *ptr - '0'; - if ((size_ & (size_ - 1)) != 0) throw std::logic_error("Size is not a valid data type size!"); - } -#endif - ++ptr; - } - } - void* get_result(uint8_t* address, const uint8_t* end) const { -#ifdef __arm64__ - if (deref_) { - auto insn = reinterpret_cast(address + offset_); - int64_t offset = 0; - bool is_sub = false, is_adrp = false; - unsigned rd = 0, sf = 0, rn = 0; - if (detail::a64_decode_b(*insn, nullptr, &offset)) { - return reinterpret_cast(reinterpret_cast(insn) + offset); - } - else if (detail::a64_decode_adr(*insn, &is_adrp, &rd, &offset)) { - auto saved_offset = offset; - auto saved_rd = rd; - auto curr_insn = insn + 1; - if (is_adrp) { - while (*curr_insn) { - unsigned rt = 0; - if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { - if (saved_rd == rd && rn == rd) { - uint64_t val = (sf ? offset : static_cast(offset)); - if (is_sub) saved_offset -= val; - else saved_offset += val; - ++curr_insn; - continue; - } - // Placing this here for sneaky programs that want to put nops in between ADRL - } else if (detail::a64_decode_nop(*curr_insn)) { - ++curr_insn; - continue; - } else if (detail::a64_decode_ldr(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_ldrh(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_str(*curr_insn, &rn, &rt, &offset)) { - if (saved_rd == rn) { - saved_offset += (offset < 0 ? -offset : offset); - } - } - break; - } - } - const auto from_addr = is_adrp ? reinterpret_cast(insn) & ~0xfff : reinterpret_cast(insn); - return reinterpret_cast(from_addr + saved_offset); - } - else if (detail::a64_decode_ldr(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_ldrh(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_str(*insn, nullptr, nullptr, &offset)) { - return reinterpret_cast(offset); - } - else if (detail::a64_decode_movz(*insn, &sf, nullptr, &offset) - || detail::a64_decode_arithmetic(*insn, nullptr, &sf, nullptr, nullptr, &offset)) { - return reinterpret_cast((sf ? offset : static_cast(offset))); - } - else - throw std::logic_error("Failed to decode instruction with defined functions"); -#else - if (deref_ || rel_) { - const auto relative_address = relative_value(address + offset_); - if (deref_) { - return reinterpret_cast(reinterpret_cast(address) - + offset_ + insn_len_ + relative_address); - } - else { - return reinterpret_cast(relative_address); - } -#endif - } - else { - return reinterpret_cast(reinterpret_cast(address) + offset_); - } - return nullptr; - } -#ifndef __arm64__ - const intptr_t relative_value(uint8_t* ptr) const { - switch (size_) { - case 1: - return static_cast(*reinterpret_cast(ptr)); - case 2: - return static_cast(*reinterpret_cast(ptr)); - case 4: - return static_cast(*reinterpret_cast(ptr)); -#if ((ULLONG_MAX) != (UINT_MAX)) - case 8: - return static_cast(*reinterpret_cast(ptr)); -#endif - } - // Should never hit here - return NULL; - } -#endif - constexpr int32_t get_inst_len_opt(const char* ptr) const { - if (*ptr > '9' || *ptr < '0') - throw std::logic_error("Invalid data for calculating remaining instruction size!"); - return detail::stoi(ptr); - } - }; -} diff --git a/README.md b/README.md index 3d48b58..43b6543 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,140 @@ -# Pattern Scanning - -#### Classes -- Pattern (base class) -- RuntimePattern -- CompileTimePattern -- XORPattern -- ArmPattern - -RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. - -CompileTimePattern creates the pattern & mask during compile time and stored in a std::array. This means the data is in the binaries module, and no pattern string left. - -XORPattern creates the pattern & mask along with a keys array during compile time. Pattern stored is XOR'd bytes, otherwise it acts just the same as the CompileTimePattern. (the pattern() method however will return the XOR'd pattern, access the original bytes with the [] operator) - -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). - -If using C++20 there is a user defined literal for the compile time pattern. - -#### Pattern Flags: -- d: Dereference from the found address, or found address + offset ( addr + size + relative address ) -- r: Read the relative value at the address, or at the address + offset (not used on arm) -- a: Perform a byte aligned scan for a faster scan. 4 byte alignment x32, 8 byte alignment x64 (rather than byte by byte scan) -- x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d - - -#### Examples: -```c++ -// Scan will read the relative value from the X offset as 4 bytes -constexpr auto compiletime_pattern = "AB CC 11 22 33 44 AB 6D X4 EF BE AD DE /r"_ctpattern; -// Scan will read the address from where the marked X is pointed to (defaults as a relative address), and perform the scan byte aligned (4 - 32bit, 8 - 64bit) -// This also uses the instruction len (notice 4 after X). This tells it during the dereference to start the RIP after the E8 instruction -constexpr auto xor_pattern = "FE ED FA CE E8 X4 ? ? ? ? EF BE AD DE /da"_xorpattern; -// Scan will read the address from where the marked X is pointed to (as a single byte; i.e. short jump) -auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern -// Scan will find the value in ADRP, and add the next ADD instruction to it (like ADRL pseudocode). Without the dereference option, this would return the address of that ADRP instruction -constexpr auto arm_pattern = ARM_PATTERN( - "LDR W?, [X?,#0x18]\n" - "X ADRP X?, #?\n" - "ADD X?, X?, #?\n" - "STR W?, [SP,#?]\n" - "/d" -) -``` \ No newline at end of file +# Pattern Scanning + +Header-only. Add `include/` to your include path and include the headers you need: +```c++ +#include +#include +``` +With CMake (C++17 or newer): +```cmake +add_subdirectory(path/to/PatternScanner) # or FetchContent_Declare + FetchContent_MakeAvailable(PatternScanner) +target_link_libraries(app PRIVATE PatternScanner::PatternScanner) +``` + +#### Classes +- Pattern (base class) +- RuntimePattern +- CompileTimePattern +- XORPattern +- ArmPattern +- ArmXORPattern +- RuntimeArmPattern + +RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. + +CompileTimePattern creates the pattern & mask during compile time and stored in a std::array. This means the data is in the binaries module, and no pattern string left. + +XORPattern creates the pattern & mask along with two keys arrays during compile time. Pattern and mask are both stored XOR'd (with separate keys), otherwise it acts just the same as the CompileTimePattern. (the pattern() and mask() methods however return the XOR'd bytes; access the original pattern bytes with the [] operator and the original mask bytes with unmask()) + +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `B.cond` (`B.?` for any condition), `BR`, `BLR`, `RET`, `RETAA`, `RETAB`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `CMP`, `CMN`, `AND(S)`, `ORR`, `EOR`, `TST`, `LDR`, `LDRB`, `LDRH`, `LDRSW`, `STR`, `STRB`, `STRH`, `LDUR`/`STUR` (and their `B`/`H`/`SW` forms), `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `NOP`, `PACIASP`, `PACIBSP`, `AUTIASP`, `AUTIBSP`. Loads and stores take W/X registers only; floating-point/SIMD ones (`LDR Q0, ...`, `STP D9, D8, ...`) are wildcards. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. `CMP`/`CMN`/`TST` are encoded as `SUBS`/`ADDS`/`ANDS` to the zero register, and an `LDR`/`STR` offset the scaled form can't hold (negative, unaligned) as `LDUR`/`STUR`, `ADD`/`SUB`/`CMP` immediates that are negative or multiples of 0x1000 as the opposite operation or `LSL #12`, and register `ADD`/`SUB` with `SP` in the extended (`UXTX`) form, as assemblers do. Other aliases aren't implemented. + +ArmXORPattern (`PatternScanner/ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. + +Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern`. Each `_xorpattern` and `_armxorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN`, `ARM_PATTERN` and `ARM_XOR_PATTERN` macros (`XOR_PATTERN` and `ARM_XOR_PATTERN` get per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). + +Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern` and their macros) are always built at compile time, so the pattern text never ends up in the binary. The class names still do, through RTTI, because `Pattern` is polymorphic. Build with `/GR-` (MSVC) or `-fno-rtti` (clang/gcc) to strip them. + +RuntimeArmPattern (`PatternScanner/RuntimeArmPattern.hpp`) is ArmPattern built at runtime, for assembly only known then (user input, config files). + +#### Target architecture +/d, /r, `X` and /a depend on the architecture of the code being scanned (`patterns::arch::x86`, `x64` or `arm64`). By default that's the one you compile for (`patterns::host_arch`); ArmPattern, ArmXORPattern and RuntimeArmPattern are always arm64. To scan code of another architecture, e.g. from a disassembler plugin, pass it to RuntimePattern: `patterns::RuntimePattern("E8 X4 ? ? ? ? /d", patterns::arch::x86)`. `resolve(match, end, &is_value)` reports whether the result is an address or a value read from the code (/r; arm64 /d on MOVZ/MOVN, ADD, or a lone load/store offset). + +arm64 /d at the X instruction: `B`/`BL`, `B.cond`, `CBZ`/`CBNZ`, `LDR` (literal) and `ADR` give their target; `ADRP` gives the page plus the `ADD`/`SUB` and load/store offset that follow it through the same register (skipping NOPs and instructions that leave the register alone); `MOVZ`/`MOVN`, `ADD`/`SUB` immediates and load/store offsets give the value. + +Scanning uses SSE2 (x86/x64) or NEON (arm64) when available. Define `PATTERNS_NO_SIMD` before including to use a portable 64-bit integer scan instead (e.g. kernel code). + +#### Wildcards: +- `?` or `??`: any byte +- `4?` / `?8`: a nibble wildcard (high nibble 4, any low nibble / any high nibble, low nibble 8) +- ArmPattern: `?` for a register (`X?`), immediate (`#?`), shift (`LSL #?`), condition (`B.?`) or a whole instruction line + +#### Pattern Flags: +- d: Dereference from the found address, or found address + offset ( addr + size + relative address ) +- r: Read the relative value at the address, or at the address + offset (not used on arm64) +- a: Only match at aligned addresses. 4 byte alignment x86 & arm64, 8 byte alignment x64 +- x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d + + +#### Examples: +```c++ +// Scan will read the relative value from the X offset as 4 bytes +constexpr auto compiletime_pattern = "AB CC 11 22 33 44 AB 6D X4 EF BE AD DE /r"_ctpattern; +// Scan will read the address from where the marked X is pointed to (defaults as a relative address), and perform the scan byte aligned (4 - 32bit, 8 - 64bit) +// This also uses the instruction len (notice 4 after X). This tells it during the dereference to start the RIP after the E8 instruction +constexpr auto xor_pattern = "FE ED FA CE E8 X4 ? ? ? ? EF BE AD DE /da"_xorpattern; +// Scan will read the address from where the marked X is pointed to (as a single byte; i.e. short jump) +auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern; +// Scan will find the value in ADRP, and add the next ADD instruction to it (like ADRL pseudocode). Without the dereference option, this would return the address of that ADRP instruction +constexpr auto arm_pattern = + "LDR W?, [X?,#0x18]\n" + "X ADRP X?, #?\n" + "ADD X?, X?, #?\n" + "STR W?, [SP,#?]\n" + "/d"_armpattern; +``` + +#### Usage +All of these are in `tests/example_usage.cpp` (styles 3-5) and `tests/test_tls.cpp` (style 6), built and checked by the tests. + +**1. One pattern, one scan** +```c++ +constexpr auto p = "48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_ctpattern; +auto ctx = p.find(start, size); +``` + +**2. Many patterns, one pass: `patterns::find_all`** +The range is walked in 1 MB chunks with every pattern run over a chunk while it's in cache, so memory is read once instead of once per pattern (~3.5x faster than a `find()` per pattern). It allocates nothing, starts no threads and needs no CRT state, so it's safe anywhere, including TLS callbacks and DllMain. +```c++ +inline constexpr auto lua_context = "48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_xorpattern; +inline constexpr auto lua_thread = "8B 0D X4 ? ? ? ? 85 C9 /d"_ctpattern; +inline constexpr const patterns::Pattern* list[] = { &lua_context, &lua_thread }; // constant-initialized + +void* out[2]; // out[i] = list[i]->find(start, size) +size_t found = patterns::find_all(start, size, list, 2, out); +``` + +**Where the module comes from.** For globals resolved at static init (3, 4), every scan gets its range from one function whose setup runs once, on first use, from whichever initializer gets there first. That's also the place to decrypt the module, or wait for it. A plain global holding the range doesn't work: the initialization order of globals across files is unspecified. +```c++ +inline std::pair game_module() { + static const auto range = [] { /* find the module, decrypt it... */ return std::pair{ base, size }; }(); + return range; +} +``` + +**3. Per global, at static init.** Simplest; one full pass over the module per global. +```c++ +// Takes the exact pattern type: MSVC rejects a compile-time-only derived temporary bound to const Pattern& (C7595) +template T scan(const P& p) { + const auto [start, size] = game_module(); + return p.template find(start, size); +} +inline lua_State** g_lua_context = scan("48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_ctpattern); +``` + +**4. Batched, at static init.** One list (X-macro); the first of its globals to initialize runs `find_all` for all of them, one pass total. Each global is still one line: `inline T name = static_cast(resolved()[name_index]);` + +**5. Explicit call.** Same list, globals start as `nullptr` and `resolve_pointers(start, size)` fills them when you choose: after decryption, from main, DllMain... + +**6. TLS callback.** Style 5 called from a TLS callback, as the module loads and before the CRT is initialized. The list, patterns and `find_all` need no runtime initialization, so they work there. + +Static init, TLS callbacks and DllMain of a DLL run under the loader lock: never start threads and wait on them there (deadlock). Outside of it, `match()` (raw match address) and `resolve()` (apply the X offset and /d /r) are public for building your own strategies, e.g. spreading chunks over threads (`tests/multi_scan.hpp`, ~10x faster again on 24 threads). + +#### Tests & benchmarks +`tests/` is a separate CMake project (the library itself stays header-only): +```sh +cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release +ctest --test-dir build -C Release --output-on-failure +./build/bench_scan [module file] # single-pattern speed; with a file, 40 real signatures sampled from it +./build/bench_multi [module file] # many patterns: find() per pattern vs find_all vs threaded +``` +- `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys. Also built with `PATTERNS_NO_SIMD`. +- `test_artifacts_*`: every compile-time pattern type (macros, and C++20 literals) must leave no pattern text in its binary, and the XOR ones no plaintext bytes, words or masks. All patterns are scanned from one function, as in a project's signature file, and built at each optimization level (`-O0`..`-O3`, `-Os`; MSVC `/Od /O1 /O2`) plus LTO, with debug info, as C++17 and C++20: optimizers fold constants differently at each. +- `test_cpp17`: the macro path. +- `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, RuntimeArmPattern parsing the same, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. +- `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. +- `test_tls` (Windows): style 6, `find_all` run from a TLS callback over the executable's own image. +- `multi_scan.hpp`: other multi-pattern strategies (per-pattern threads, chunks over threads) for the benchmark. + +#### Plugins +- [`plugins/ida`](plugins/ida/README.md): search IDA databases with these patterns (IDA 9.x, Windows/macOS/Linux). diff --git a/RuntimePattern.hpp b/RuntimePattern.hpp deleted file mode 100644 index 38decf3..0000000 --- a/RuntimePattern.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once -#include "Pattern.hpp" -#include -#include -#include - -namespace patterns { - - class RuntimePattern : public Pattern { - std::vector m_pattern; - std::vector m_mask; - public: - RuntimePattern(const char* p, size_t len) { - int n = 0; - for (int i = 0; i < len; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern.push_back(value(ptr)); - m_mask.push_back(0xFF); - ++n; - } - else i--; - } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#endif - length_ = n; - } - RuntimePattern(const char* p) : - RuntimePattern(p, strlen(p)) - { - } - ~RuntimePattern() = default; - virtual const uint8_t* pattern() const override { - return m_pattern.data(); - } - virtual const uint8_t* mask() const override { - return m_mask.data(); - } - }; -} - -auto operator"" _rtpattern(const char* str, size_t size) { - return patterns::RuntimePattern(str, size); -} \ No newline at end of file diff --git a/XORPattern.hpp b/XORPattern.hpp deleted file mode 100644 index 41ddc16..0000000 --- a/XORPattern.hpp +++ /dev/null @@ -1,117 +0,0 @@ -#pragma once -#include "CompileTimePattern.hpp" -#include - -namespace patterns { - namespace detail { - // Used to generate the pattern string hash - constexpr uint32_t fnv1a(const char* s, size_t count) { - return ((count ? fnv1a(s, count - 1) : 2166136261u) ^ s[count]) * 16777619u; - } - - template - constexpr __forceinline uint32_t fnv1a(const char(&s)[count]) { - return fnv1a(s, count - 1); - } - - template - constexpr __forceinline uint8_t generate_key() { - uint32_t value = 2166136261u + seed; - for (char c : __TIME__) - value = static_cast((value ^ c) * 16777619ull); - return value; - } - - template - constexpr auto generate_keys(std::index_sequence) { - return std::array{ detail::generate_key()...}; - } - } - - template - class XORPattern : public Pattern { - std::array pattern_; - std::array mask_; - std::array keys_; - public: - constexpr XORPattern(const char* p) : - pattern_{}, mask_{}, keys_{ detail::generate_keys(std::make_index_sequence()) } - { - length_ = narr; - auto n = 0; - for (auto i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - pattern_[n] = 0 ^ keys_[n]; - mask_[n] = 0; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - pattern_[n] = value(ptr) ^ keys_[n]; - mask_[n] = 0xFF; - ++n; - } - else --i; - } - } - virtual const uint8_t* pattern() const override { - return pattern_.data(); - } - virtual const uint8_t* mask() const override { - return mask_.data(); - } - // Override on [] operator to perform XOR for original byte - virtual uint8_t operator[](size_t idx) const override { - return pattern_[idx] ^ keys_[idx]; - } - virtual void* find(const uint8_t* bytes, size_t size) const override { - void* result = nullptr; - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; - for (auto j = 0U; j < length_; ++j) { - if (mask_[j] == 0xFF && (*this)[j] != i[j]) { - found = false; - break; - } - } - if (found) - return get_result(i, end); - } - return result; - } - template - T find(const uint8_t* bytes, size_t size) const { - return reinterpret_cast(find(bytes, size)); - } - }; - -} - -#if __cplusplus > 201703L -template -constexpr auto operator"" _xorpattern() { - return patterns::XORPattern(str.data); -} -#else -#ifndef XOR_PATTERN -#define XOR_PATTERN(x) patterns::XORPattern(x) -#endif -#endif \ No newline at end of file diff --git a/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp similarity index 51% rename from ArmPattern.hpp rename to include/PatternScanner/ArmPattern.hpp index 5f10d53..dcb5bf6 100644 --- a/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -1,8 +1,8 @@ #pragma once #include #include -#include #include +#include #include "Pattern.hpp" @@ -54,6 +54,8 @@ namespace patterns { return i; } + constexpr size_t max_line = 128; + constexpr bool starts_with(const char* a, const char* prefix) { while (*prefix) { if (*a != *prefix) return false; @@ -99,6 +101,15 @@ namespace patterns { return 0xFFFFFFFF; } + // 1 for a 64-bit register (X, SP, XZR), 0 for 32-bit (W, WSP, WZR), -1 when unknown ("?") + constexpr int register_width(const char* token) { + return token[0] == 'X' || str_eq(token, "SP") ? 1 : token[0] == 'W' ? 0 : -1; + } + + constexpr bool is_sp(const char* token) { + return str_eq(token, "SP") || str_eq(token, "WSP"); + } + template constexpr void parse_register(char(&buffer)[N], uint32_t mask, uint32_t shift, ArmOpcode& base) { if (auto val = parse_register(buffer); val != 0xFFFFFFFF) @@ -127,7 +138,7 @@ namespace patterns { constexpr size_t skip_mnemonic(const char* instr) { size_t i = 0; - while (instr[i] && instr[i] != ' ') ++i; + while (instr[i] && !isspace(instr[i])) ++i; return i; } @@ -148,22 +159,36 @@ namespace patterns { return (v >> (n & 63)) | (v << (-n & 63)); } + // std::countr_zero / countl_zero are C++20; the macros need C++17 + constexpr int countr_zero(uint64_t n) { + int c = 0; + for (; c < 64 && !(n >> c & 1); ++c) {} + return c; + } + constexpr int countl_zero(uint64_t n) { + int c = 0; + for (; c < 64 && !(n >> (63 - c) & 1); ++c) {} + return c; + } + constexpr uint64_t clear_trailing_ones(uint64_t n) { return n & (n + 1); } - template - constexpr std::tuple encode_bitmask_immediate(T value) { - uint64_t val = (value > 0xFFFFFFFFULL) ? value : (static_cast(static_cast(value)) << 32) | static_cast(value); + // 32-bit (W) immediates are encoded as the value repeated in both halves; 64-bit (X) ones as they are + constexpr std::tuple encode_bitmask_immediate(uint64_t value, bool is64) { + if (!is64 && value > 0xFFFFFFFFULL) + return { false, 0, 0, 0 }; + uint64_t val = is64 ? value : (value << 32) | value; if (val == 0 || ~val == 0) return { false, 0, 0, 0 }; - int rotation = std::countr_zero(clear_trailing_ones(val)); + int rotation = countr_zero(clear_trailing_ones(val)); uint64_t normalized = rotate_right(val, rotation & 63); - int zeroes = std::countl_zero(normalized); - int ones = std::countr_zero(~normalized); + int zeroes = countl_zero(normalized); + int ones = countr_zero(~normalized); int size = zeroes + ones; if (rotate_right(val, size & 63) != val) @@ -196,7 +221,7 @@ namespace patterns { } constexpr ArmOpcode parse_branch_imm26(const char* instr, ArmOpcode base) { - size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + size_t i = skip_char(instr, skip_whitespace(instr, skip_mnemonic(instr)), '#'); if (instr[i] == '?') { base.pattern &= 0xFC000000; @@ -208,6 +233,14 @@ namespace patterns { return base; } + // BR/BLR/RET Xn (a bare RET returns through X30) + constexpr ArmOpcode parse_branch_reg(const char* instr, ArmOpcode base) { + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + if (instr[i]) parse_register(instr, i, 0x1F, 5, base); + else base.set_bits(30, 0x1F, 5); + return base; + } + constexpr ArmOpcode parse_adr(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); // Rd @@ -216,25 +249,44 @@ namespace patterns { char imm[24] = {}; i = parse_imm_token(instr, skip_char(instr, i, '#'), imm); if (!(imm[0] == '?' && imm[1] == '\0')) { - if (int64_t offset = stoi(imm); offset != -1) { - if ((base.pattern >> 31) & 1) offset >>= 12; - base.set_bits(static_cast((offset << 24) | ((offset >> 2) & 0x7FFFF)), (3 << 24 | 0x7FFFF), 5); - } + int64_t offset = stoi(imm); + if ((base.pattern >> 31) & 1) offset >>= 12; + base.set_bits(static_cast((offset << 24) | ((offset >> 2) & 0x7FFFF)), (3 << 24 | 0x7FFFF), 5); } return base; } - constexpr std::tuple parse_load_store_defaults(const char* instr, size_t i, ArmOpcode& base, uint32_t size_bit = 0) { + // imm19 (bits 5-23) from a byte offset, for CBZ/CBNZ, B.cond and LDR (literal) + constexpr void parse_imm19(const char* instr, size_t i, ArmOpcode& base) { + char imm[24] = {}; + parse_imm_token(instr, i, imm); + if (imm[0] != '?') base.set_bits(static_cast(stoi(imm) >> 2), 0x7FFFF, 5); + } + + // Single-register loads/stores. op is the unsigned offset form; the unscaled (LDUR), pre/post-index and + // register offset forms differ from it in fixed bits. scale is log2 of the access size, or -1 to take it + // from Rt (W or X, bit 30). An offset the unsigned form can't hold (negative, unaligned) is encoded unscaled, + // the way assemblers pick LDUR/STUR. Floating-point/SIMD registers aren't supported: the line is a wildcard + template + constexpr ArmOpcode parse_load_store(const char* instr, ArmOpcode) { + ArmOpcode base{ 0, 0 }; + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + char rt[5] = {}, imm[24] = {}; i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); - if (size_bit && rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, size_bit); - if (instr[i++] != '[') return { false, false, false, false }; + if (rt[0] != 'W' && rt[0] != 'X' && rt[0] != '?') return { 0, 0 }; + int size = scale; + if (scale < 0 && rt[0] != '?') { + size = rt[0] == 'W' ? 2 : 3; + base.set_bits(size - 2, 1, 30); + } + if (instr[i++] != '[') return { 0, 0 }; // Rn i = parse_register(instr, i, 0x1F, 5, base); bool is_register_offset = false, is_pre_index = false, is_post_index = false; - uint32_t option_val = 0b011, s_val = 0; + uint32_t option_val = 0b011, s_val = 0, s_mask = 1; if (instr[i] == ',') { i = skip_whitespace(instr, ++i); @@ -252,7 +304,9 @@ namespace patterns { i = skip_whitespace(instr, skip_char(instr, i, ',')); if (instr[i] == '#') { - s_val = static_cast(stoi(&instr[++i])) > 0 ? 1 : 0; + // For bytes an explicit #0 is the shifted form; for the rest the amount is 0 or the scale + if (instr[++i] == '?') s_mask = 0; + else s_val = (size == 0 || stoi(&instr[i]) != 0) ? 1 : 0; } } } @@ -268,72 +322,56 @@ namespace patterns { } } - if (is_register_offset) base.set_bits((option_val << 1) | s_val, 0xF, 12); - - if (imm[0] != '?') { - int32_t val = stoi(imm); - if (!is_register_offset && (is_pre_index || is_post_index)) { - if (val >= -256 && val <= 255) - base.set_bits(val, 0x1FF, 12); - } else if (!is_register_offset && (size_bit == 0 || rt[0] != '?')) { - auto shift = size_bit > 0 ? (rt[0] == 'W' ? 2 : 3) : 1; - if ((val & 1) == 0 && (val >> shift) <= 0xFFF) { - base.set_bits((val >> shift), 0xFFF, 10); - } + const bool known_imm = imm[0] != '?'; + const int32_t val = known_imm ? stoi(imm) : 0; + bool is_unsigned_offset = !unscaled && !is_register_offset && !is_pre_index && !is_post_index; + if (is_unsigned_offset && known_imm && (val < 0 || (size > 0 && (val & ((1 << size) - 1))))) + is_unsigned_offset = false; + + const uint32_t width_mask = scale < 0 ? ~(1u << 30) : ~0u; // Rt's width bit, set above when known + if (is_unsigned_offset) { + base.set_bits(op, 0xFFC00000 & width_mask); + if (known_imm && size >= 0) { + if ((val >> size) > 0xFFF) return { 0, 0 }; + base.set_bits(val >> size, 0xFFF, 10); } + return base; } - return { true, is_register_offset, is_pre_index, is_post_index }; - } - - constexpr ArmOpcode parse_ldrh(const char* instr, ArmOpcode base) { - size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base); - if (!ok) return base; - - if (is_register_offset) base.set_bits(0x78600800, 0xFFE00C00); - else if (is_pre_index) base.set_bits(0x78400C00, 0xFFE00C00); - else if (is_post_index) base.set_bits(0x78400400, 0xFFE00C00); - else /* Unsigned offset*/ base.set_bits(0x79400000, 0xFFC00000); - + const uint32_t form = is_register_offset ? 0x00200800 : is_pre_index ? 0xC00 : is_post_index ? 0x400 : 0; + base.set_bits((op & ~0x01000000u) | form, 0xFFE00C00 & width_mask); + if (is_register_offset) { + base.set_bits(option_val << 1 | s_val, 0xE | s_mask, 12); + } else if (known_imm) { + if (val < -256 || val > 255) return { 0, 0 }; + base.set_bits(val, 0x1FF, 12); + } return base; } + // LDR is a load/store, or a PC-relative literal load (LDR Rt, #offset) constexpr ArmOpcode parse_ldr(const char* instr, ArmOpcode base) { + char rt[5] = {}; size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base, 30); - if (!ok) { - char imm[24] = {}; - i = parse_imm_token(instr, skip_whitespace(instr, skip_past(instr, i, ',')), imm); - - if (!(imm[0] == '?' && imm[1] == '\0')) { - int32_t val = stoi(imm); - if ((val & 3) != 0) val &= ~3; - base.set_bits((val >> 2), 0x7FFFF, 5); - } - return base; - } - - if (is_register_offset) base.set_bits(0xB8600800, 0xBFE00C00); - else if (is_pre_index) base.set_bits(0xB8400C00, 0xBFE00C00); - else if (is_post_index) base.set_bits(0xB8400400, 0xBFE00C00); - else base.set_bits(0xB9400000, 0xBFC00000); - + i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); + if (instr[i] == '[') return parse_load_store<0xB9400000, -1>(instr, base); + if (rt[0] != 'W' && rt[0] != 'X' && rt[0] != '?') return { 0, 0 }; + if (rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, 30); + parse_imm19(instr, i, base); return base; } constexpr ArmOpcode parse_orr(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - char rd[5] = {}/*, rn[5] = {}*/; + char rd[5] = {}, rn[5] = {}; i = parse_register(instr, parse_token(instr, i, rd), rd, 0x1F, 0, base, ','); - if (rd[0] != '?') base.set_bits((rd[0] == 'W' ? 0 : 1), 1, 31); - // Rn - i = parse_register(instr, i, 0x1F, 5, base, ','); - + i = parse_register(instr, parse_token(instr, i, rn), rn, 0x1F, 5, base, ','); + // Width from Rd, or from Rn when Rd is "?" + const int width = register_width(rd) >= 0 ? register_width(rd) : register_width(rn); + if (width >= 0) base.set_bits(static_cast(width), 1, 31); + if (instr[i] == '#') { base.set_bits((2 << 4), (3 << 4 | 1), 23); @@ -341,8 +379,11 @@ namespace patterns { i = parse_imm_token(instr, i, imm); if (imm[0] != '?') { uint64_t immVal = stoi(imm); - auto [ ok, N, imms, immr ] = encode_bitmask_immediate(immVal); - if (ok) base.set_bits((N << 12 | immr << 6 | imms), (1 << 12 | 0x3F << 6 | 0x3F), 10); + // Unknown width (Rd is "?"): a value that fits in 32 bits can be either, and the two only differ in N + const bool is64 = width == 1 || (width < 0 && immVal > 0xFFFFFFFFULL); + const uint32_t n_mask = (width >= 0 || is64) ? 1 : 0; + auto [ ok, N, imms, immr ] = encode_bitmask_immediate(immVal, is64); + if (ok) base.set_bits((N << 12 | immr << 6 | imms), (n_mask << 12 | 0x3F << 6 | 0x3F), 10); } return base; } @@ -359,7 +400,7 @@ namespace patterns { else if (starts_with(&instr[i], "ROR")) { base.set_bits(3, 3, 22); i += 3; } else base.set_bits(0, 3, 22); - i = skip_whitespace(instr, skip_char(instr, i, '#')); + i = skip_char(instr, skip_whitespace(instr, i), '#'); if (instr[i] != '?') base.set_bits(stoi(&instr[i]), 0x3F, 10); } else { @@ -372,29 +413,44 @@ namespace patterns { constexpr ArmOpcode parse_arithmetic(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - char rd[5] = {}; + char rd[5] = {}, rn[5] = {}; i = parse_register(instr, parse_token(instr, i, rd), rd, 0x1F, 0, base, ','); - if (rd[0] != '?') base.set_bits((rd[0] == 'W' ? 0b0 : 0b1), 1, 31); - // Rn - i = parse_register(instr, i, 0x1F, 5, base, ','); + i = parse_register(instr, parse_token(instr, i, rn), rn, 0x1F, 5, base, ','); + // Width from Rd, or from Rn when Rd is "?" + const int width = register_width(rd) >= 0 ? register_width(rd) : register_width(rn); + if (width >= 0) base.set_bits(static_cast(width), 1, 31); if (instr[i] == '#') { base.set_bits((2 << 4 | 2), (3 << 4 | 7), 23); char imm[24] = {}; - i = parse_imm_token(instr, i, imm); - if (imm[0] != '?') base.set_bits(stoi(imm), 0xFFF, 10); - + i = skip_whitespace(instr, parse_imm_token(instr, i, imm)); + int sh = -1; // LSL #12 (1) or #0 (0) when written, -1 if not or #? if (instr[i] == ',') { - i = skip_whitespace(instr, i+1); - if (starts_with(&instr[i], "LSL#0")) base.set_mask(1, 22); - else if (starts_with(&instr[i], "LSL#12")) base.set_bits(1, 1, 22); - } else { - base.set_mask(1, 22); + i = skip_whitespace(instr, i + 1); + if (starts_with(&instr[i], "LSL")) { + i = skip_char(instr, skip_whitespace(instr, i + 3), '#'); + if (instr[i] != '?') sh = stoi(&instr[i]) == 12 ? 1 : 0; + } + } + if (imm[0] != '?') { + int64_t val = stoi(imm); + // As assemblers do: a negative immediate is the opposite operation (ADD #-8 is SUB #8, CMP #-1 + // is CMN #1), and a multiple of 0x1000 the shifted form (#0x1000 is #1, LSL #12) + if (val < 0) { base.pattern ^= 1u << 30; val = -val; } + if (sh < 0 && val > 0xFFF && !(val & 0xFFF)) { sh = 1; val >>= 12; } + if (val > 0xFFF) return { 0, 0 }; + base.set_bits(static_cast(val), 0xFFF, 10); + if (sh < 0) sh = 0; } + if (sh >= 0) base.set_bits(static_cast(sh), 1, 22); return base; } + // With SP as Rd or Rn only the extended register form exists: assemblers write LSL as UXTX (UXTW for W) + const bool sp = is_sp(rd) || is_sp(rn); + const uint32_t sp_extend = str_eq(rd, "WSP") || str_eq(rn, "WSP") ? 2 : 3; + base.set_bits(5, 0xF, 25); // Rm i = parse_register(instr, i, 0x1F, 16, base); @@ -402,7 +458,8 @@ namespace patterns { if (instr[i] == ',') { i = skip_whitespace(instr, i + 1); bool shifted = false, extended = false; - if (starts_with(&instr[i], "LSL")) { base.set_bits(0, 7, 21); shifted = true; i += 3; } + if (starts_with(&instr[i], "LSL") && sp) { base.set_bits(sp_extend, 7, 13); extended = true; i += 3; } + else if (starts_with(&instr[i], "LSL")) { base.set_bits(0, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "LSR")) { base.set_bits(2, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "ASR")) { base.set_bits(4, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "UXTB")) { base.set_bits(0, 7, 13); extended = true; i += 4; } @@ -437,8 +494,10 @@ namespace patterns { base.set_bits((1 << 11), (7 << 11 | 7), 10); } } + } else if (sp) { + base.set_bits((1 << 11 | sp_extend << 3), (7 << 11 | 0x3F), 10); } else { - base.set_bits(0, (7 << 11 | 0x7F), 10); + base.set_bits(0, (7 << 11 | 0x3F), 10); } return base; @@ -448,25 +507,26 @@ namespace patterns { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); char rt1[5] = {}; i = parse_register(instr, parse_token(instr, i, rt1), rt1, 0x1F, 0, base, ','); - bool know_size = rt1[0] != '?'; - uint32_t size_bits = know_size ? (rt1[0] == 'W' ? 0b0 : 0b1) : 0; + if (rt1[0] != 'W' && rt1[0] != 'X' && rt1[0] != '?') return { 0, 0 }; // FP/SIMD pairs aren't supported + char rt2[5] = {}; + i = parse_register(instr, parse_token(instr, i, rt2), rt2, 0x1F, 10, base, ','); + // Width from Rt1, or from Rt2 when Rt1 is "?" + const int width = register_width(rt1) >= 0 ? register_width(rt1) : register_width(rt2); + const bool know_size = width >= 0; + const uint32_t size_bits = know_size ? static_cast(width) : 0; if (know_size) base.set_bits(size_bits, 1, 31); - // Rt2 - i = parse_register(instr, i, 0x1F, 10, base, ','); - if (instr[i] != '[') return base; i = skip_whitespace(instr, skip_char(instr, i, '[')); // Rn i = parse_register(instr, i, 0x1F, 5, base, ','); - bool post_index = false, signed_offset = false, pre_index = false; + // [Rn] alone is the signed offset form with offset 0 (imm stays empty, which reads as 0) + bool post_index = false, pre_index = false; char imm[24] = {}; - if (instr[i] == '#') { + if (instr[i] == '#') i = skip_whitespace(instr, parse_imm_token(instr, i, imm)); - signed_offset = true; - } if (instr[i] == ']') { i = skip_whitespace(instr, skip_char(instr, skip_char(instr, i, ']'), ',')); if (instr[i] == '!') { @@ -476,7 +536,6 @@ namespace patterns { i = parse_imm_token(instr, i, imm); } } - if (!(post_index || pre_index || signed_offset)) return base; base.set_bits((post_index ? 0b0010100011 : pre_index ? 0b0010100111 : 0b0010100101), 0x1FF, 22); @@ -493,18 +552,65 @@ namespace patterns { return base; } - constexpr ArmOpcode parse_str(const char* instr, ArmOpcode base) { - size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + // STP is LDP with the load bit (22) clear + constexpr ArmOpcode parse_stp(const char* instr, ArmOpcode base) { + base = parse_ldp(instr, base); + base.pattern &= ~(1u << 22); + return base; + } - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base, 30); - if (!ok) return base; + // MOV is an alias; encoded the way assemblers pick: ADD #0 when SP is involved, ORR from the zero register for + // registers, and MOVZ, then MOVN, then ORR (bitmask) for immediates. MOV Rd, #? matches MOVZ or MOVN + constexpr ArmOpcode parse_mov(const char* instr, ArmOpcode) { + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + char rd[5] = {}; + i = skip_whitespace(instr, skip_char(instr, parse_token(instr, i, rd), ',')); + const bool rd_sp = str_eq(rd, "SP") || str_eq(rd, "WSP"); + const bool known_width = rd[0] == 'W' || rd[0] == 'X' || rd_sp; + const bool is64 = rd[0] == 'X' || str_eq(rd, "SP"); + ArmOpcode op{ 0, 0 }; + if (known_width) op.set_bits(is64 ? 1 : 0, 1, 31); + if (auto r = parse_register(rd); r != 0xFFFFFFFF) op.set_bits(r, 0x1F); - if (is_register_offset) base.set_bits(0xB8200800, 0xBFE00C00); - else if (is_pre_index) base.set_bits(0xB8000C00, 0xBFE00C00); - else if (is_post_index) base.set_bits(0xB8000400, 0xBFE00C00); - else base.set_bits(0xB9000000, 0xBFC00000); + if (instr[i] == '#') { + char imm[24] = {}; + parse_imm_token(instr, i, imm); + if (imm[0] == '?') { + op.set_bits(0x12800000, 0x3F800000); // MOVZ or MOVN: bit 30 of opc left open, bit 29 clear (not MOVK) + return op; + } + const unsigned width = is64 ? 64 : 32; + const uint64_t ones = is64 ? ~0ULL : 0xFFFFFFFFULL; + const uint64_t v = stoi(imm) & ones; // unsigned: 64-bit hex fits, and negatives wrap + for (uint32_t hw = 0; hw < width / 16; ++hw) { + if ((v & ~(0xFFFFULL << (hw * 16))) == 0) { + op.set_bits(0x52800000 | hw << 21 | static_cast(v >> (hw * 16)) << 5, 0x7FFFFFE0); + return op; + } + } + for (uint32_t hw = 0; hw < width / 16; ++hw) { + if ((~v & ones & ~(0xFFFFULL << (hw * 16))) == 0) { + op.set_bits(0x12800000 | hw << 21 | static_cast((~v & ones) >> (hw * 16)) << 5, 0x7FFFFFE0); + return op; + } + } + auto [ ok, N, imms, immr ] = encode_bitmask_immediate(v, is64); + if (!ok) return { 0, 0 }; + op.set_bits(0x320003E0 | N << 22 | immr << 16 | imms << 10, 0x7FFFFFE0); + return op; + } - return base; + char rm[5] = {}; + parse_token(instr, i, rm); + const auto rm_val = parse_register(rm); + if (rd_sp || str_eq(rm, "SP") || str_eq(rm, "WSP")) { + op.set_bits(0x11000000, 0x7FFFFC00); // ADD Rd, Rn, #0 + if (rm_val != 0xFFFFFFFF) op.set_bits(rm_val, 0x1F, 5); + } else { + op.set_bits(0x2A0003E0, 0x7FE0FFE0); // ORR Rd, ZR, Rm + if (rm_val != 0xFFFFFFFF) op.set_bits(rm_val, 0x1F, 16); + } + return op; } constexpr ArmOpcode parse_cbz(const char* instr, ArmOpcode base) { @@ -514,14 +620,39 @@ namespace patterns { i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); if (rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, 31); - char imm[24] = {}; - i = parse_imm_token(instr, i, imm); + parse_imm19(instr, i, base); + return base; + } - if (!(imm[0] == '?' && imm[1] == '\0')) { - int32_t val = stoi(imm); - if ((val & 3) != 0) val &= ~3; - base.set_bits((val >> 2), 0x7FFFF, 5); + // CMP/CMN/TST Rn, ... are SUBS/ADDS/ANDS ZR, Rn, ...: parsed as that, with the zero register of Rn's width + template + constexpr ArmOpcode parse_zr_dest(const char* instr, ArmOpcode base) { + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + const char* zr = instr[i] == 'W' ? "WZR, " : instr[i] == 'X' ? "XZR, " : "?, "; + char line[max_line + 8] = { '_', ' ' }; + size_t j = 2; + while (*zr) line[j++] = *zr++; + while (instr[i]) line[j++] = instr[i++]; + ArmOpcode op = parse(line, base); + op.set_bits(31, 0x1F); + return op; + } + + constexpr const char* conditions[] = { "EQ", "NE", "CS", "CC", "MI", "PL", "VS", "VC", "HI", "LS", "GE", "LT", "GT", "LE", "AL", "NV" }; + + // B.cond: the condition is part of the mnemonic (B.EQ, B.HS); B.? matches any + constexpr ArmOpcode parse_bcond(const char* instr) { + ArmOpcode base{ 0x54000000, 0xFF000010 }; + char cond[4] = {}; + for (size_t j = 0; j < 3 && instr[2 + j] && !isspace(instr[2 + j]); ++j) cond[j] = instr[2 + j]; + if (cond[0] != '?') { + uint32_t c = str_eq(cond, "HS") ? 2 : str_eq(cond, "LO") ? 3 : 16; + for (uint32_t k = 0; k < 16 && c == 16; ++k) + if (str_eq(cond, conditions[k])) c = k; + if (c == 16) return { 0, 0 }; + base.set_bits(c, 0xF); } + parse_imm19(instr, skip_whitespace(instr, skip_mnemonic(instr)), base); return base; } @@ -535,20 +666,42 @@ namespace patterns { { "BL", parse_branch_imm26 }, { "ADR", parse_adr }, { "ADRP", parse_adr }, - { "LDRH", parse_ldrh }, { "LDR", parse_ldr }, + { "LDRB", parse_load_store<0x39400000, 0> }, + { "LDRH", parse_load_store<0x79400000, 1> }, + { "LDRSW", parse_load_store<0xB9800000, 2> }, + { "STR", parse_load_store<0xB9000000, -1> }, + { "STRB", parse_load_store<0x39000000, 0> }, + { "STRH", parse_load_store<0x79000000, 1> }, + { "LDUR", parse_load_store<0xB9400000, -1, true> }, + { "LDURB", parse_load_store<0x39400000, 0, true> }, + { "LDURH", parse_load_store<0x79400000, 1, true> }, + { "LDURSW", parse_load_store<0xB9800000, 2, true> }, + { "STUR", parse_load_store<0xB9000000, -1, true> }, + { "STURB", parse_load_store<0x39000000, 0, true> }, + { "STURH", parse_load_store<0x79000000, 1, true> }, { "MOVZ", parse_movX }, { "MOVN", parse_movX }, { "MOVK", parse_movX }, { "ORR", parse_orr }, + { "AND", parse_orr }, + { "ANDS", parse_orr }, + { "EOR", parse_orr }, + { "TST", parse_zr_dest }, + { "CMP", parse_zr_dest }, + { "CMN", parse_zr_dest }, { "LDP", parse_ldp }, + { "STP", parse_stp }, + { "MOV", parse_mov }, { "ADD", parse_arithmetic }, { "ADDS", parse_arithmetic }, { "SUB", parse_arithmetic }, { "SUBS", parse_arithmetic }, - { "STR", parse_str }, { "CBZ", parse_cbz }, { "CBNZ", parse_cbz }, + { "BR", parse_branch_reg }, + { "BLR", parse_branch_reg }, + { "RET", parse_branch_reg }, }; constexpr InstructionDescriptor instructions[] = { @@ -560,20 +713,31 @@ namespace patterns { { "ADDS", { 0x21000000, 0x65800000 } }, { "SUB", { 0x41000000, 0x65000000 } }, { "SUBS", { 0x61000000, 0x65000000 } }, - { "LDRH", { 0x38400000, 0x3B200000 } }, { "LDR", { 0x18000000, 0xBF000000 } }, { "MOVZ", { 0x52800000, 0x7F800000 } }, { "MOVN", { 0x12800000, 0x7F800000 } }, { "MOVK", { 0x72800000, 0x7F800000 } }, { "ORR", { 0x22000000, 0x67000000 } }, + { "AND", { 0x02000000, 0x67000000 } }, + { "ANDS", { 0x62000000, 0x67000000 } }, + { "EOR", { 0x42000000, 0x67000000 } }, + { "TST", { 0x62000000, 0x67000000 } }, // ANDS ZR + { "CMP", { 0x61000000, 0x65000000 } }, // SUBS ZR + { "CMN", { 0x21000000, 0x65800000 } }, // ADDS ZR { "LDP", { 0x28400000, 0x7E400000 } }, + { "STP", { 0x28000000, 0x7E400000 } }, { "BR", { 0xD61F0000, 0xFFFFFC1F } }, { "BLR", { 0xD63F0000, 0xFFFFFC1F } }, - { "STR", { 0xB8000000, 0xBEC00000 } }, { "CBZ", { 0x34000000, 0x7F000000 } }, { "CBNZ", { 0x35000000, 0x7F000000 } }, { "NOP", { 0xD503201F, 0xFFFFFFFF } }, - { "RET", { 0xD65F03C0, 0xFFFFFFFF } }, + { "RET", { 0xD65F0000, 0xFFFFFC1F } }, + { "RETAA", { 0xD65F0BFF, 0xFFFFFFFF } }, + { "RETAB", { 0xD65F0FFF, 0xFFFFFFFF } }, + { "PACIASP", { 0xD503233F, 0xFFFFFFFF } }, + { "PACIBSP", { 0xD503237F, 0xFFFFFFFF } }, + { "AUTIASP", { 0xD50323BF, 0xFFFFFFFF } }, + { "AUTIBSP", { 0xD50323FF, 0xFFFFFFFF } }, }; constexpr ArmOpcode find_opcode(const char* mnemonic) { @@ -588,12 +752,13 @@ namespace patterns { constexpr ArmOpcode a64_encode_instruction(const char* instr) { char mnemonic[8] = {}; size_t i = 0; - while (instr[i] && instr[i] != ' ' && i < 7) { + while (instr[i] && !isspace(instr[i]) && i < 7) { mnemonic[i] = instr[i]; ++i; } mnemonic[i] = '\0'; + if (starts_with(mnemonic, "B.")) return parse_bcond(instr); ArmOpcode base = find_opcode(mnemonic); for (const auto& entry : parseTable) { @@ -605,14 +770,43 @@ namespace patterns { return base; } - constexpr size_t total_instructions(const char* str, size_t nstr) { - size_t line = 0, count = 0; - for (size_t i = 0; i < nstr - 1; ++i) { - if (str[i] == '\n') ++line, count = 0; - else if (count == 0 && str[i] == '/') break; - else ++count; + // Copies the line at str[i] into out, trimmed and uppercased (lldb/objdump print lowercase), so each + // instruction is parsed on its own and can't run into the next line. Returns where the next line starts + template + constexpr size_t read_line(const char* str, size_t i, char (&out)[N]) { + while (str[i] == ' ' || str[i] == '\t') ++i; + size_t j = 0; + for (; str[i] && str[i] != '\n'; ++i) { + if (j + 1 >= N) throw std::logic_error("Instruction line is too long!"); + out[j++] = (str[i] >= 'a' && str[i] <= 'z') ? static_cast(str[i] - 'a' + 'A') : str[i]; } - return line + (count > 0 ? 1 : 0); + while (j && isspace(out[j - 1])) --j; + out[j] = '\0'; + return str[i] ? i + 1 : i; + } + + // The ArmPattern syntax: one instruction per line, X before the one to resolve, flags after '/'. Calls + // line(index, instruction) for each instruction (trimmed and uppercased), x(index) for the X line and + // flags(text) for the text after '/'. Reads up to n chars or a terminator. Returns the number of instructions + template + constexpr size_t parse_arm(const char* p, size_t n, Line&& line, X&& x, Flags&& flags) { + const size_t options = skip_past(p, 0, '/'); + if (options && p[options - 1] == '/') flags(&p[options]); + size_t count = 0; + for (size_t i = 0; i < n && p[i];) { + char text[max_line] = {}; + i = read_line(p, i, text); + if (text[0] == '/') break; + if (!text[0]) continue; + size_t start = 0; + if (text[0] == 'X' && (!text[1] || isspace(text[1]))) { x(count); start = skip_whitespace(text, 1); } + line(count++, &text[start]); + } + return count; + } + + constexpr size_t total_instructions(const char* str, size_t nstr) { + return parse_arm(str, nstr - 1, [](size_t, const char*) {}, [](size_t) {}, [](const char*) {}); } template @@ -622,29 +816,24 @@ namespace patterns { } template - class ArmPattern : public Pattern { + class ArmPattern final : public Pattern { std::array m_instructions{}; std::array m_masks{}; public: - constexpr ArmPattern(const char* p) : + PATTERNS_CONSTEVAL ArmPattern(const char* p) : m_instructions{}, m_masks{} { + arch_ = arch::arm64; length_ = narr * 4; - uint32_t idx = 0; - // Find the options - size_t options = detail::skip_past(p, 0, '/'); - if (p[options - 1] == '/') handle_options(&p[options]); align_ = true; - for (size_t i = detail::skip_whitespace(p, 0); i < nstr && idx < narr;) { - size_t start = detail::skip_char(p, i, ' '); - if (p[start] == 'X') { offset_ = idx * 4; start = detail::skip_whitespace(p, start + 1); } - size_t end = detail::skip_past(p, start, '\n'); - if (idx + 1 == narr && end > options) end = options; - auto encoded = detail::a64_encode_instruction(&p[start]); - m_instructions[idx] = encoded.pattern; - m_masks[idx++] = encoded.mask; - i = end; - } + detail::parse_arm(p, nstr, + [&](size_t i, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions[i] = encoded.pattern; + m_masks[i] = encoded.mask; + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); } virtual const uint8_t* pattern() const override { return reinterpret_cast(m_instructions.data()); @@ -655,6 +844,13 @@ namespace patterns { }; } +#ifdef PATTERNS_LITERALS +template +PATTERNS_CONSTEVAL auto operator""_armpattern() { + return patterns::ArmPattern(str.data); +} +#endif + #ifndef ARM_PATTERN -#define ARM_PATTERN(x) patterns::ArmPattern(x) +#define ARM_PATTERN(x) ([] { constexpr auto p = patterns::ArmPattern(x); return p; }()) #endif \ No newline at end of file diff --git a/include/PatternScanner/ArmXORPattern.hpp b/include/PatternScanner/ArmXORPattern.hpp new file mode 100644 index 0000000..f0d0fa8 --- /dev/null +++ b/include/PatternScanner/ArmXORPattern.hpp @@ -0,0 +1,85 @@ +#pragma once +#include "ArmPattern.hpp" +#include "XORPattern.hpp" +#include + +namespace patterns { + + // ArmPattern with the instruction words and masks stored XOR'd, like XORPattern + template + class ArmXORPattern final : public Pattern { + std::array m_instructions{}; + std::array m_masks{}; + std::array m_keys{}; + std::array m_mask_keys{}; + public: + PATTERNS_CONSTEVAL ArmXORPattern(const char* p, uint32_t seed) : + m_instructions{}, m_masks{}, m_keys{}, m_mask_keys{} + { + arch_ = arch::arm64; + length_ = narr * 4; + align_ = true; + for (size_t j = 0; j < narr; ++j) { + for (uint32_t b = 0; b < 4; ++b) { + const auto at = seed + static_cast(j * 4 + b); + m_keys[j] |= static_cast(detail::generate_key(at)) << (b * 8); + m_mask_keys[j] |= static_cast(detail::generate_key(at, detail::mask_keys)) << (b * 8); + } + } + // Encoded here at compile time; only the XOR'd words make it into the binary + detail::parse_arm(p, nstr, + [&](size_t i, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions[i] = encoded.pattern ^ m_keys[i]; + m_masks[i] = encoded.mask ^ m_mask_keys[i]; + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); + } + virtual const uint8_t* pattern() const override { + return reinterpret_cast(m_instructions.data()); + } + // XOR'd like pattern(); unmask() gives the original + virtual const uint8_t* mask() const override { + return reinterpret_cast(m_masks.data()); + } + uint8_t unmask(size_t idx) const { + return byte(m_masks[idx / 4] ^ m_mask_keys[idx / 4], idx); + } + // Override on [] operator to perform XOR for original byte + virtual uint8_t operator[](size_t idx) const override { + return byte(m_instructions[idx / 4] ^ m_keys[idx / 4], idx); + } + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const override { + // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd words + std::array plain{}, plain_mask{}; + // Keys read through volatile so the optimizer can't fold the XOR and emit the plaintext (see XORPattern) + const volatile uint32_t* keys = m_keys.data(); + const volatile uint32_t* mask_keys = m_mask_keys.data(); + for (size_t j = 0; j < narr; ++j) { + plain[j] = m_instructions[j] ^ keys[j]; + plain_mask[j] = m_masks[j] ^ mask_keys[j]; + } + return scan(bytes, size, reinterpret_cast(plain.data()), + reinterpret_cast(plain_mask.data())); + } + private: + // Byte idx % 4 of a word, in memory order like pattern() and mask() + static uint8_t byte(uint32_t word, size_t idx) { + uint8_t bytes[4]; + memcpy(bytes, &word, 4); + return bytes[idx % 4]; + } + }; +} + +#ifdef PATTERNS_LITERALS +template +PATTERNS_CONSTEVAL auto operator""_armxorpattern() { + return patterns::ArmXORPattern(str.data, str.seed); +} +#endif + +#ifndef ARM_XOR_PATTERN +#define ARM_XOR_PATTERN(x) ([] { constexpr auto p = patterns::ArmXORPattern(x, patterns::detail::site_seed(__FILE__, __LINE__, __COUNTER__)); return p; }()) +#endif diff --git a/include/PatternScanner/CompileTimePattern.hpp b/include/PatternScanner/CompileTimePattern.hpp new file mode 100644 index 0000000..c92d336 --- /dev/null +++ b/include/PatternScanner/CompileTimePattern.hpp @@ -0,0 +1,47 @@ +#pragma once +#include "Pattern.hpp" +#include + +namespace patterns { + + namespace detail { + constexpr size_t pattern_length(const char* s, size_t nstr) { + return parse_bytes(s, nstr - 1, [](size_t, char, char) {}, [](size_t, const char*) {}, [](const char*) {}); + } + + template + constexpr size_t pattern_length(const char(&s)[nstr]) { + return pattern_length(s, nstr); + } + } + + template + class CompileTimePattern final : public Pattern { + std::array m_pattern; + std::array m_mask; + public: + PATTERNS_CONSTEVAL CompileTimePattern(const char* p) : + m_pattern{}, m_mask{} + { + length_ = narr; + parse(p, nstr, [&](size_t n, uint8_t byte, uint8_t mask) { m_pattern[n] = byte; m_mask[n] = mask; }); + } + virtual const uint8_t* pattern() const override { + return m_pattern.data(); + } + virtual const uint8_t* mask() const override { + return m_mask.data(); + } + }; +} + +#ifdef PATTERNS_LITERALS +template +PATTERNS_CONSTEVAL auto operator""_ctpattern() { + return patterns::CompileTimePattern(str.data); +} +#endif + +#ifndef COMPILETIME_PATTERN +#define COMPILETIME_PATTERN(x) ([] { constexpr auto p = patterns::CompileTimePattern(x); return p; }()) +#endif diff --git a/include/PatternScanner/Pattern.hpp b/include/PatternScanner/Pattern.hpp new file mode 100644 index 0000000..aa7ca9c --- /dev/null +++ b/include/PatternScanner/Pattern.hpp @@ -0,0 +1,612 @@ +#pragma once +#include +#include +#include +#include +#include + +// Define PATTERNS_NO_SIMD where vector registers are off-limits (e.g. kernel code); find() then uses 64-bit integer (SWAR) scanning +#if defined(PATTERNS_NO_SIMD) +#elif (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) && !defined(_M_ARM64EC) +#include +#define PATTERNS_SSE2 +#elif defined(__ARM_NEON) || defined(_M_ARM64) +#include +#define PATTERNS_NEON +#endif + +// Compiling for arm64: patterns default to arm64 semantics (/d decoding, 4-byte alignment). __arm64__ alone is +// Apple only: GCC and clang elsewhere define __aarch64__, MSVC _M_ARM64 +#if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) +#define PATTERNS_ARM64 +#endif + +#ifndef _MSC_VER +#define __forceinline inline __attribute__((always_inline)) +#endif + +// Compile-time patterns must never be built at runtime: that would put the pattern text in the binary. +// consteval enforces it (C++20); before that the macros force it with a constexpr local. +#ifdef __cpp_consteval +#define PATTERNS_CONSTEVAL consteval +#else +#define PATTERNS_CONSTEVAL constexpr +#endif + +// String literal operators (_ctpattern, _xorpattern, _armpattern) need C++20. MSVC reports __cplusplus as +// 199711L unless /Zc:__cplusplus is passed, so check _MSVC_LANG as well +#if __cplusplus > 201703L || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) +#define PATTERNS_LITERALS +#if __has_include() +#include +#endif +#endif + +namespace patterns { + namespace detail { + constexpr uint32_t hash(const char* s, uint32_t h = 2166136261u) { + while (*s) + h = (h ^ static_cast(*s++)) * 16777619u; + return h; + } + + // Unique per definition site (file + line + column/counter), so identical patterns get different XOR keys + constexpr uint32_t site_seed(const char* file, uint32_t line, uint32_t column) { + return hash(file) ^ line * 0x9E3779B1u ^ column * 0x85EBCA77u; + } + +#ifdef PATTERNS_LITERALS + // Literal operator template argument. The defaulted source_location is evaluated where the literal is + // written, which is how "..."_xorpattern gets per-definition keys without a macro + template + struct const_string { + char data[nstr]{}; + uint32_t seed = 0; + static constexpr size_t length = nstr - 1; + static constexpr size_t size = nstr; + +#ifdef __cpp_lib_source_location + PATTERNS_CONSTEVAL const_string(const char(&s)[nstr], std::source_location loc = std::source_location::current()) + : seed(site_seed(loc.file_name(), loc.line(), loc.column())) { +#else + // No source_location (older standard library): keys still vary per pattern and per build, not per site + PATTERNS_CONSTEVAL const_string(const char(&s)[nstr]) : seed(hash(s)) { +#endif + for (size_t i = 0; i < nstr; ++i) + data[i] = s[i]; + } + }; +#endif + constexpr bool __forceinline is_digit(char c) { + return c <= '9' && c >= '0'; + } + constexpr bool __forceinline is_hex_digit(char c) { + return is_digit(c) || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); + } + constexpr __forceinline char get_bits(char c) { + return is_digit(c) ? (c - '0') : ((c & (~0x20)) - 'A' + 0xA); + } + // Decimal or 0x hex, optionally negative; stops at the first other character. Accumulates unsigned, so long + // numbers wrap instead of overflowing + template + constexpr T stoi(const char* str) { + using U = std::make_unsigned_t; + const bool negative = *str == '-'; + if (negative) ++str; + const bool hex = str[0] == '0' && (str[1] == 'x' || str[1] == 'X'); + if (hex) str += 2; + U value = 0; + for (; hex ? is_hex_digit(*str) : is_digit(*str); ++str) + value = value * (hex ? 16 : 10) + static_cast(get_bits(*str)); + return static_cast(negative ? U(0) - value : value); + } + + // Masked compare, 8 bytes at a time (memcpy keeps unaligned loads legal; compilers emit a single mov) + inline bool masked_equal(const uint8_t* at, const uint8_t* pat, const uint8_t* msk, size_t len) { + size_t j = 0; + for (; j + 8 <= len; j += 8) { + uint64_t d, p, m; + memcpy(&d, at + j, 8); memcpy(&p, pat + j, 8); memcpy(&m, msk + j, 8); + if ((d ^ p) & m) return false; + } + for (; j < len; ++j) + if ((at[j] ^ pat[j]) & msk[j]) return false; + return true; + } + + // The byte pattern syntax ("48 8B ? X4 ? ? ? ? /d"). Calls byte(index, hi, lo) for each byte (its two characters: + // hex digits or '?'; lo is '\0' past the end), x(index, len_text) for the X marker (len_text at the digits after + // X, or nullptr) and flags(text) for the text after '/'. Returns the number of bytes + template + constexpr size_t parse_bytes(const char* p, size_t len, Byte&& byte, X&& x, Flags&& flags) { + size_t n = 0; + for (size_t i = 0; i < len; i += 2) { + const char c = p[i]; + if (c == 'X' || c == 'x') { + x(n, i + 1 < len && p[i + 1] != ' ' ? &p[i + 1] : nullptr); + while (i + 1 < len && p[i + 1] != ' ') + ++i; + } + // The flags end the pattern + else if (c == '/') { + flags(&p[i + 1]); + break; + } + else if (c != ' ') + byte(n++, c, i + 1 < len ? p[i + 1] : '\0'); + else --i; + } + return n; + } + + template + T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { + constexpr size_t int_width = sizeof(int32_t) * 8; + return (static_cast(insn) << (int_width - (offset + width))) >> (int_width - width); + } + + bool __forceinline decode_masked_match(uint32_t insn, uint32_t mask, uint32_t pattern) { + return (insn & mask) == pattern; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/NOP--No-Operation- + inline bool a64_decode_nop(uint32_t insn) { + if (insn == 0b1101'0101'0000'0011'0010'0000'0001'1111) + return true; + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/B--Branch- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/BL--Branch-with-Link- + inline bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { + // x001 01?? ???? ???? ???? ???? ???? ???? + // 1 - BL + // 0 - B + if (decode_masked_match(insn, 0b0111'1100'0000'0000'0000'0000'0000'0000, 0b0001'0100'0000'0000'0000'0000'0000'0000)) { + if (is_bl) *is_bl = (insn >> 31) & 0x1; + *offset = extract_bitfield(insn, 26, 0) << 2; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADR--Form-PC-relative-address- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- + inline bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { + // rd - destination register + // x??1 0000 ???? ???? ???? ???? ???? ???? + // 1 - ADRP + // 0 - ADR + // bits 30 & 29 - immlo + // bits 23 -> 5 - immhi + // bits 4 -> 0 - rd + if (decode_masked_match(insn, 0b0001'1111'0000'0000'0000'0000'0000'0000, 0b0001'0000'0000'0000'0000'0000'0000'0000)) { + if (rd) *rd = insn & 0x1f; + uint32_t immlo = (insn >> 29) & 0x3; + int32_t immhi = extract_bitfield(insn, 19, 5) << 2; + auto adrp = (insn >> 31) & 0x1; + if (is_adrp) *is_adrp = adrp; + // Back to signed (immlo is unsigned) before widening, and a 64-bit multiply: ADRP reaches +-4GB + const int64_t imm = static_cast(immhi | immlo); + *offset = (adrp ? imm * 4096 : imm); + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Index-by-Encoding/Loads-and-Stores + // Single general-purpose register loads/stores with an immediate offset (LDR/STR/LDRB/LDRH/LDRSW... and the + // LDUR/STUR forms). *offset is signed; *post_index is set for [Rn],#imm, which accesses Rn itself + inline bool a64_decode_load_store(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset, bool* post_index = nullptr) { + // size 111 0 01 opc imm12 Rn Rt: unsigned offset, scaled by the access size + if (decode_masked_match(insn, 0x3F000000, 0x39000000)) { + *offset = static_cast(extract_bitfield(insn, 12, 10)) << (insn >> 30); + if (post_index) *post_index = false; + } + // size 111 0 00 opc 0 imm9 mode Rn Rt: unscaled (00), post-index (01), unprivileged (10), pre-index (11) + else if (decode_masked_match(insn, 0x3F200000, 0x38000000)) { + *offset = extract_bitfield(insn, 9, 12); + if (post_index) *post_index = ((insn >> 10) & 3) == 1; + } + else return false; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + + // B.cond, CBZ/CBNZ and LDR (literal): a PC-relative imm19 + inline bool a64_decode_imm19(uint32_t insn, int64_t* offset) { + if (decode_masked_match(insn, 0xFF000010, 0x54000000) || decode_masked_match(insn, 0x7E000000, 0x34000000) + || decode_masked_match(insn, 0x3F000000, 0x18000000)) { + *offset = static_cast(extract_bitfield(insn, 19, 5)) * 4; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVN--Move-wide-with-NOT- + inline bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { + // sf - 1 for 64 bit, 0 for 32 bit + // rd - destination register + // sx01 0010 1hh? ???? ???? ???? ???? ???? + // s: 1 = 64 bit, 0 = 32 bit. x: 1 = MOVZ, 0 = MOVN (bit 29 clear excludes MOVK). hh: shift / 16 + if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0010'1000'0000'0000'0000'0000'0000)) { + const auto hw = (insn >> 21) & 3; + const uint64_t imm = static_cast(extract_bitfield(insn, 16, 5)) << (hw * 16); + *offset = static_cast((insn >> 30) & 1 ? imm : ~imm); + if (sf) *sf = (insn >> 31) & 0x1; + if (rd) *rd = insn & 0x1f; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADD--immediate---Add--immediate-- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/SUB--immediate---Subtract--immediate-- + inline bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { + // rd - destination register + // rn - source register + // sf - 1 for 64 bit, 0 for 32 bit + // ?x01 0001 0??? ???? ???? ???? ???? ???? + // 1 = subtraction + // 0 = addition + if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0001'0000'0000'0000'0000'0000'0000)) { + if (sf) *sf = (insn >> 31) & 0x1; + if (rd) *rd = insn & 0x1f; + if (rn) *rn = (insn >> 5) & 0x1f; + if (is_sub) *is_sub = (insn >> 30) & 0x1; + uint32_t imm12 = extract_bitfield(insn, 12, 10); + // sh = 1 is LSL#12 + auto sh = (insn >> 22) & 0x1; + *offset = (sh ? (imm12 << 12) : imm12); + return true; + } + return false; + } + } + + // Instruction set a pattern's /d /r, X and /a follow. Defaults to the one being compiled for; RuntimePattern + // takes another, to scan code of a different architecture (e.g. from a disassembler plugin) + enum class arch : uint8_t { x86, x64, arm64 }; +#ifdef PATTERNS_ARM64 + constexpr arch host_arch = arch::arm64; +#else + constexpr arch host_arch = sizeof(void*) == 8 ? arch::x64 : arch::x86; +#endif + + class Pattern { + protected: + uint32_t length_ = 0; + uint32_t offset_ = 0; + bool deref_ = false; + // x86/x64 only. Remaining bytes from the offset when used in dereferencing + uint32_t insn_len_ = 0; + // x86/x64 only (arm64 instructions are 32 bit and encoded, so arm64 has just deref). Defaulting 4 byte + // relative address reading + uint32_t size_ = 4; + bool rel_ = false; + bool align_ = false; + // Set before parsing: the flags and X are read per architecture + arch arch_ = host_arch; + // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level + constexpr size_t align_size() const { + return arch_ == arch::x64 ? 8 : 4; + } + // Patterns are used through Pattern* but never deleted through one; protected keeps the destructor + // trivial (constexpr patterns in C++17) while making `delete` through a Pattern* a compile error + ~Pattern() = default; + public: + Pattern() = default; + constexpr uint32_t length() const { + return length_; + } + constexpr uint32_t offset() const { + return offset_; + } + constexpr bool deref() const { + return deref_; + } + constexpr bool relative() const { + return rel_; + } + constexpr bool aligned() const { + return align_; + } + constexpr arch target() const { + return arch_; + } + virtual const uint8_t* mask() const = 0; + virtual const uint8_t* pattern() const = 0; + template + T find(const uint8_t* bytes, size_t size) const { + return reinterpret_cast(find(bytes, size)); + } + void* find(const uint8_t* bytes, size_t size) const { + const auto hit = match(bytes, size); + return hit ? resolve(hit, bytes + size) : nullptr; + } + // Address of the first match, before the X offset and /d /r are applied + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const { + return scan(bytes, size, pattern(), mask()); + } + // What find() returns for a match from match(). end bounds how far /d may read (arm64 ADRP sequences). + // *is_value is set when the result is a value read from the code (/r; arm64 /d on LDR/STR offsets, MOVZ, + // ADD) rather than an address + void* resolve(const uint8_t* match, const uint8_t* end, bool* is_value = nullptr) const { + return get_result(const_cast(match), end, is_value); + } + virtual uint8_t operator[](size_t idx) const { + return pattern()[idx]; + } + protected: + // Parses a byte pattern: sets the X offset, X and flags, and calls emit(index, byte, mask) for each byte. + // Set arch_ first. Returns the number of bytes + template + constexpr size_t parse(const char* p, size_t len, Emit&& emit) { + return detail::parse_bytes(p, len, + [&](size_t n, char hi, char lo) { + // A lone ? (or ??) is a whole byte; a ? next to a hex digit is a nibble ("4?", "?8") + if (hi == '?' && lo != '?' && !detail::is_hex_digit(lo)) lo = '?'; + emit(n, uint8_t(nibble(hi) << 4 | nibble(lo)), uint8_t((hi == '?' ? 0 : 0xF0) | (lo == '?' ? 0 : 0x0F))); + }, + [&](size_t n, const char* len_text) { + offset_ = static_cast(n); + if (len_text && arch_ != arch::arm64) + insn_len_ = get_inst_len_opt(len_text); + }, + [&](const char* flags) { handle_options(flags); }); + } + // First match of pat/msk in [bytes, bytes + size). Two anchor bytes are checked first + // (16 positions at a time with SIMD), and only positions passing both get a full compare. + const uint8_t* scan(const uint8_t* bytes, size_t size, const uint8_t* pat, const uint8_t* msk) const { + if (size < length_) return nullptr; + if (!length_) return bytes; + const size_t last = size - length_; + const size_t step = align_ ? align_size() : 1; + // Aligned scans align to the absolute address, not to wherever the caller's buffer starts + size_t i = (step - reinterpret_cast(bytes) % step) % step; + // Anchors: first and last of the most specific bytes (most mask bits set). Far apart, so random + // data rarely satisfies both at once + size_t a1 = 0, a2 = 0; + int best = 0; + for (size_t j = 0; j < length_; ++j) { + int bits = 0; + for (unsigned m = msk[j]; m; m &= m - 1) ++bits; + if (bits > best) { best = bits; a1 = a2 = j; } + else if (bits && bits == best) a2 = j; + } + // All wildcards leaves m1 = m2 = 0, so every position passes the filter and gets compared + const uint8_t m1 = msk[a1], p1 = pat[a1] & m1, m2 = msk[a2], p2 = pat[a2] & m2; +#if defined(PATTERNS_SSE2) + const auto vm1 = _mm_set1_epi8(static_cast(m1)), vp1 = _mm_set1_epi8(static_cast(p1)); + const auto vm2 = _mm_set1_epi8(static_cast(m2)), vp2 = _mm_set1_epi8(static_cast(p2)); + auto block = [&](size_t at) { + const auto v1 = _mm_loadu_si128(reinterpret_cast(bytes + at + a1)); + const auto v2 = _mm_loadu_si128(reinterpret_cast(bytes + at + a2)); + return static_cast(static_cast(_mm_movemask_epi8(_mm_and_si128( + _mm_cmpeq_epi8(_mm_and_si128(v1, vm1), vp1), _mm_cmpeq_epi8(_mm_and_si128(v2, vm2), vp2))))); + }; + // 64 positions per branch (measured ~1.4x over 16 on real x64 code), then 16 at a time for the remainder + for (; i + 63 <= last; i += 64) { + const uint64_t hits = block(i) | block(i + 16) << 16 | block(i + 32) << 32 | block(i + 48) << 48; + if (hits) + for (size_t k = 0; k < 64; k += step) + if ((hits >> k & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } + for (; i + 15 <= last; i += 16) { + const auto hits = block(i); + if (hits) + for (size_t k = 0; k < 16; k += step) + if ((hits >> k & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } +#elif defined(PATTERNS_NEON) + const auto vm1 = vdupq_n_u8(m1), vp1 = vdupq_n_u8(p1), vm2 = vdupq_n_u8(m2), vp2 = vdupq_n_u8(p2); + for (; i + 15 <= last; i += 16) { + const auto eq = vandq_u8(vceqq_u8(vandq_u8(vld1q_u8(bytes + i + a1), vm1), vp1), + vceqq_u8(vandq_u8(vld1q_u8(bytes + i + a2), vm2), vp2)); + // NEON has no movemask; a narrowing shift packs lane k into bits 4k..4k+3 + const uint64_t hits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); + if (hits) + for (size_t k = 0; k < 16; k += step) + if ((hits >> (k * 4) & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } +#else + // No vector registers: same filter on 8 positions at once in a plain 64-bit integer (SWAR) + constexpr uint64_t ones = 0x0101010101010101ULL; + for (; i + 7 <= last; i += 8) { + uint64_t v1, v2; + memcpy(&v1, bytes + i + a1, 8); + memcpy(&v2, bytes + i + a2, 8); + // Byte k of x is zero when position i + k passes both anchors + const uint64_t x = ((v1 & m1 * ones) ^ p1 * ones) | ((v2 & m2 * ones) ^ p2 * ones); + // High bit set for zero bytes (may also flag a byte above a zero; masked_equal weeds those out) + const uint64_t hits = (x - ones) & ~x & 0x8080808080808080ULL; + if (hits) + for (size_t k = 0; k < 8; k += step) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + const auto lane = 7 - k; +#else + const auto lane = k; +#endif + if ((hits >> (lane * 8 + 7) & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } + } +#endif + // Remainder that doesn't fill a full block + for (; i <= last; i += step) + if ((bytes[i + a1] & m1) == p1 && (bytes[i + a2] & m2) == p2 && detail::masked_equal(bytes + i, pat, msk, length_)) + return bytes + i; + return nullptr; + } + // Credits to EJT for the helper functions here! + static constexpr uint8_t nibble(char c) { + if (c == '?') return 0; + if (!detail::is_hex_digit(c)) + throw std::logic_error("Invalid hex byte in pattern!"); + return static_cast(detail::get_bits(c)); + } + constexpr void handle_options(const char* ptr) { + const bool arm64 = arch_ == arch::arm64; + while (*ptr) { + if (*ptr == 'd') { + if (rel_) throw std::logic_error("Cannot use relative and deref together!"); + deref_ = true; + } + else if (*ptr == 'r' && !arm64) { + if (deref_) throw std::logic_error("Cannot use relative and deref together!"); + rel_ = true; + } + else if (*ptr == 'a') + align_ = true; + // Check the next character to see what size we're reading at this relative address + else if (!arm64 && *ptr > '0' && (arch_ == arch::x64 ? *ptr < '9' : *ptr < '5')) { + size_ = *ptr - '0'; + if ((size_ & (size_ - 1)) != 0) throw std::logic_error("Size is not a valid data type size!"); + } + ++ptr; + } + } + void* get_result(uint8_t* address, const uint8_t* end, bool* is_value = nullptr) const { + if (is_value) *is_value = false; + if (arch_ == arch::arm64 && deref_) { + auto insn = reinterpret_cast(address + offset_); + int64_t offset = 0; + bool is_sub = false, is_adrp = false; + unsigned rd = 0, sf = 0, rn = 0; + if (detail::a64_decode_b(*insn, nullptr, &offset)) { + return reinterpret_cast(reinterpret_cast(insn) + offset); + } + else if (detail::a64_decode_adr(*insn, &is_adrp, &rd, &offset)) { + auto saved_offset = offset; + auto saved_rd = rd; + auto curr_insn = insn + 1; + if (is_adrp) { + // Follow the instructions after ADRP, never reading past the end of the scanned buffer. + // Instructions that leave the register alone (NOP, another ADRP, unrelated ADD/MOVZ/loads/ + // stores; compilers interleave them) are skipped; anything else ends the chain + for (; reinterpret_cast(curr_insn + 1) <= end && *curr_insn; ++curr_insn) { + unsigned rt = 0; + bool post_index = false; + if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { + // ADD may write a different register (ADRP X8 / ADD X0, X8, #lo); keep following that one + if (rn == saved_rd) { + uint64_t val = (sf ? offset : static_cast(offset)); + if (is_sub) saved_offset -= val; + else saved_offset += val; + saved_rd = rd; + continue; + } + if (rd == saved_rd) break; + } + else if (detail::a64_decode_load_store(*curr_insn, &rn, &rt, &offset, &post_index)) { + // The address it accesses; [Rn],#imm accesses Rn and only then adds + if (rn == saved_rd) { + if (!post_index) saved_offset += offset; + break; + } + if (rt == saved_rd) break; + } + else if (detail::a64_decode_adr(*curr_insn, nullptr, &rd, &offset) + || detail::a64_decode_movz(*curr_insn, nullptr, &rd, &offset)) { + if (rd == saved_rd) break; + } + else if (!detail::a64_decode_nop(*curr_insn)) + break; + } + } + const auto from_addr = is_adrp ? reinterpret_cast(insn) & ~0xfff : reinterpret_cast(insn); + return reinterpret_cast(from_addr + saved_offset); + } + else if (detail::a64_decode_imm19(*insn, &offset)) { + return reinterpret_cast(reinterpret_cast(insn) + offset); + } + else if (detail::a64_decode_load_store(*insn, nullptr, nullptr, &offset)) { + if (is_value) *is_value = true; + return reinterpret_cast(offset); + } + else if (detail::a64_decode_movz(*insn, &sf, nullptr, &offset) + || detail::a64_decode_arithmetic(*insn, nullptr, &sf, nullptr, nullptr, &offset)) { + if (is_value) *is_value = true; + return reinterpret_cast((sf ? offset : static_cast(offset))); + } + else + throw std::logic_error("Failed to decode instruction with defined functions"); + } + else if (arch_ != arch::arm64 && (deref_ || rel_)) { + const auto relative_address = relative_value(address + offset_); + if (deref_) { + return reinterpret_cast(reinterpret_cast(address) + + offset_ + insn_len_ + relative_address); + } + else { + if (is_value) *is_value = true; + return reinterpret_cast(relative_address); + } + } + else { + return reinterpret_cast(reinterpret_cast(address) + offset_); + } + return nullptr; + } + // memcpy: operands in x86 code are rarely aligned for their type + template + static intptr_t read_signed(const uint8_t* ptr) { + T v; + memcpy(&v, ptr, sizeof v); + return static_cast(v); + } + intptr_t relative_value(uint8_t* ptr) const { + switch (size_) { + case 1: + return read_signed(ptr); + case 2: + return read_signed(ptr); + case 4: + return read_signed(ptr); +#if UINTPTR_MAX > 0xFFFFFFFF + case 8: + return read_signed(ptr); +#endif + } + // Should never hit here + return 0; + } + constexpr int32_t get_inst_len_opt(const char* ptr) const { + if (*ptr > '9' || *ptr < '0') + throw std::logic_error("Invalid data for calculating remaining instruction size!"); + return detail::stoi(ptr); + } + }; + + // Resolves many patterns over one range: out[i] = pats[i]->find(start, size). Returns how many matched. + // Faster than a find() per pattern: the range is walked in cache-sized chunks with every pattern run over a + // chunk while it's hot, so memory is read once instead of once per pattern. + // No allocation, threads, CRT state or static initialization, so it's safe from TLS callbacks and DllMain. + inline size_t find_all(const uint8_t* start, size_t size, const Pattern* const* pats, size_t count, void** out, + size_t chunk = 1 << 20) { + if (!chunk) chunk = 1 << 20; + size_t found = 0; + for (size_t i = 0; i < count; ++i) out[i] = nullptr; + // out[i] holds the raw match until the end. Chunks go in order, so the first hit is the lowest address + for (size_t pos = 0; pos < size && found < count; pos += chunk) { + for (size_t i = 0; i < count; ++i) { + if (out[i]) continue; + // Reach length - 1 bytes into the next chunk, so matches straddling the boundary are found, + // but no match starting in the next chunk is + const size_t n = chunk + pats[i]->length() - 1; + if (auto hit = pats[i]->match(start + pos, n < size - pos ? n : size - pos)) { + out[i] = const_cast(hit); + ++found; + } + } + } + for (size_t i = 0; i < count; ++i) + if (out[i]) out[i] = pats[i]->resolve(static_cast(out[i]), start + size); + return found; + } +} diff --git a/include/PatternScanner/RuntimeArmPattern.hpp b/include/PatternScanner/RuntimeArmPattern.hpp new file mode 100644 index 0000000..c7650f1 --- /dev/null +++ b/include/PatternScanner/RuntimeArmPattern.hpp @@ -0,0 +1,40 @@ +#pragma once +#include "ArmPattern.hpp" +#include +#include + +namespace patterns { + + // ArmPattern built at runtime, for text only known then (user input, config files). Same syntax and results; + // like RuntimePattern, the text is in the binary if it's a literal. + class RuntimeArmPattern final : public Pattern { + std::vector m_instructions; + std::vector m_masks; + public: + RuntimeArmPattern(const char* text, size_t len) { + arch_ = arch::arm64; + align_ = true; + // The parser reads up to a terminator, so work on a terminated copy + const std::string s(text, len); + const auto count = detail::parse_arm(s.c_str(), len, + [&](size_t, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions.push_back(encoded.pattern); + m_masks.push_back(encoded.mask); + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); + length_ = static_cast(count * 4); + } + explicit RuntimeArmPattern(const char* text) : + RuntimeArmPattern(text, strlen(text)) + { + } + virtual const uint8_t* pattern() const override { + return reinterpret_cast(m_instructions.data()); + } + virtual const uint8_t* mask() const override { + return reinterpret_cast(m_masks.data()); + } + }; +} diff --git a/include/PatternScanner/RuntimePattern.hpp b/include/PatternScanner/RuntimePattern.hpp new file mode 100644 index 0000000..f2a33c4 --- /dev/null +++ b/include/PatternScanner/RuntimePattern.hpp @@ -0,0 +1,37 @@ +#pragma once +#include "Pattern.hpp" +#include +#include +#include + +namespace patterns { + + class RuntimePattern final : public Pattern { + std::vector m_pattern; + std::vector m_mask; + public: + // target: the architecture of the code being scanned, for /d /r, X and /a (see patterns::arch) + RuntimePattern(const char* p, size_t len, arch target = host_arch) { + arch_ = target; + length_ = static_cast(parse(p, len, [&](size_t, uint8_t byte, uint8_t mask) { + m_pattern.push_back(byte); + m_mask.push_back(mask); + })); + } + RuntimePattern(const char* p, arch target = host_arch) : + RuntimePattern(p, strlen(p), target) + { + } + ~RuntimePattern() = default; + virtual const uint8_t* pattern() const override { + return m_pattern.data(); + } + virtual const uint8_t* mask() const override { + return m_mask.data(); + } + }; +} + +inline auto operator""_rtpattern(const char* str, size_t size) { + return patterns::RuntimePattern(str, size); +} \ No newline at end of file diff --git a/include/PatternScanner/XORPattern.hpp b/include/PatternScanner/XORPattern.hpp new file mode 100644 index 0000000..2f88d85 --- /dev/null +++ b/include/PatternScanner/XORPattern.hpp @@ -0,0 +1,78 @@ +#pragma once +#include "CompileTimePattern.hpp" +#include + +namespace patterns { + namespace detail { + // Key byte from the definition site's seed, mixed with the build time so keys also change every build. + // Pattern and mask keys come from different streams: with shared keys, pattern ^ mask would cancel them out + enum key_stream : uint8_t { pattern_keys, mask_keys }; + constexpr uint8_t generate_key(uint32_t seed, key_stream stream = pattern_keys) { + uint32_t value = static_cast(((2166136261u + seed) ^ stream) * 16777619ull); + for (char c : __TIME__) + value = static_cast((value ^ c) * 16777619ull); + return static_cast(value); + } + } + + template + class XORPattern final : public Pattern { + std::array pattern_; + std::array mask_; + std::array keys_; + std::array mask_keys_; + public: + PATTERNS_CONSTEVAL XORPattern(const char* p, uint32_t seed) : + pattern_{}, mask_{}, keys_{}, mask_keys_{} + { + for (size_t j = 0; j < narr; ++j) { + keys_[j] = detail::generate_key(seed + static_cast(j)); + mask_keys_[j] = detail::generate_key(seed + static_cast(j), detail::mask_keys); + } + length_ = narr; + parse(p, nstr, [&](size_t n, uint8_t byte, uint8_t mask) { + pattern_[n] = byte ^ keys_[n]; + mask_[n] = mask ^ mask_keys_[n]; + }); + } + virtual const uint8_t* pattern() const override { + return pattern_.data(); + } + // XOR'd like pattern(); unmask() gives the original + virtual const uint8_t* mask() const override { + return mask_.data(); + } + uint8_t unmask(size_t idx) const { + return mask_[idx] ^ mask_keys_[idx]; + } + // Override on [] operator to perform XOR for original byte + virtual uint8_t operator[](size_t idx) const override { + return pattern_[idx] ^ keys_[idx]; + } + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const override { + // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd bytes + std::array plain{}, plain_mask{}; + // Keys read through volatile: otherwise, with a constexpr pattern, the optimizer folds the XOR at compile + // time and emits the plaintext as a constant (seen with clang -O3) + const volatile uint8_t* keys = keys_.data(); + const volatile uint8_t* mask_keys = mask_keys_.data(); + for (size_t j = 0; j < narr; ++j) { + plain[j] = pattern_[j] ^ keys[j]; + plain_mask[j] = mask_[j] ^ mask_keys[j]; + } + return scan(bytes, size, plain.data(), plain_mask.data()); + } + }; + +} + +#ifdef PATTERNS_LITERALS +template +PATTERNS_CONSTEVAL auto operator""_xorpattern() { + return patterns::XORPattern(str.data, str.seed); +} +#endif + +#ifndef XOR_PATTERN +#define XOR_PATTERN(x) ([] { constexpr auto p = patterns::XORPattern(x, patterns::detail::site_seed(__FILE__, __LINE__, __COUNTER__)); return p; }()) +#endif \ No newline at end of file diff --git a/plugins/ida/CMakeLists.txt b/plugins/ida/CMakeLists.txt new file mode 100644 index 0000000..04d5e69 --- /dev/null +++ b/plugins/ida/CMakeLists.txt @@ -0,0 +1,67 @@ +# PatternScanner IDA plugin (IDA 9.x). Needs the IDA SDK with ida-cmake, which isn't part of this repo: +# cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk [-DIDABIN=/path/to/ida] +# cmake --build build-ida --config Release +# The plugin is copied to $IDABIN/plugins (IDABIN defaults to the SDK's bin folder). +# +# Optional idalib test of the search, against small ELF files assembled with clang and lld: +# -DIDA_INSTALL_DIR=/path/to/ida then ctest --test-dir build-ida -C Release --output-on-failure +cmake_minimum_required(VERSION 3.27) +project(PatternScannerIDA CXX) +set(CMAKE_CXX_STANDARD 17) + +set(IDASDK "$ENV{IDASDK}" CACHE PATH "IDA SDK: a GitHub checkout (ida-cmake in src/cmake) or a zip SDK with ida-cmake cloned into it") +if(NOT IDASDK) + message(FATAL_ERROR "Set IDASDK (-DIDASDK=... or the environment variable) to your IDA SDK") +endif() +# ida-cmake reads the environment variable +set(ENV{IDASDK} "${IDASDK}") +find_file(IDA_CMAKE_BOOTSTRAP bootstrap.cmake + PATHS "${IDASDK}/src/cmake" "${IDASDK}/cmake" "${IDASDK}/ida-cmake" NO_DEFAULT_PATH NO_CACHE) +if(NOT IDA_CMAKE_BOOTSTRAP) + message(FATAL_ERROR "ida-cmake not found in ${IDASDK}. GitHub SDK: git submodule update --init src/cmake. " + "Zip SDK: git clone https://github.com/allthingsida/ida-cmake ${IDASDK}/ida-cmake") +endif() +include(${IDA_CMAKE_BOOTSTRAP}) +# The bootstrap points the environment variable at src/ for GitHub checkouts; idasdk reads this variable first +set(IDASDK "$ENV{IDASDK}") +find_package(idasdk REQUIRED) + +add_subdirectory(../.. PatternScanner) + +ida_add_plugin(pattern_scanner + SOURCES pattern_scanner.cpp + LIBRARIES PatternScanner::PatternScanner +) + +set(IDA_INSTALL_DIR "" CACHE PATH "IDA install with idalib and a license; builds the search test") +if(IDA_INSTALL_DIR) + find_program(CLANG NAMES clang) + find_program(LLD NAMES ld.lld ld.lld-21 ld.lld-20 ld.lld-19 ld.lld-18) + if(NOT CLANG OR NOT LLD) + message(FATAL_ERROR "The search test assembles its fixtures with clang and ld.lld") + endif() + set(fixtures) + foreach(f x64:x86_64:x86.S x86:i386:x86.S arm64:aarch64:arm64.S) + string(REPLACE ":" ";" f "${f}") + list(GET f 0 name) + list(GET f 1 triple) + list(GET f 2 src) + set(out ${CMAKE_CURRENT_BINARY_DIR}/fixture_${name}.elf) + add_custom_command(OUTPUT ${out} + COMMAND ${CLANG} --target=${triple}-linux-gnu -nostdlib -static --ld-path=${LLD} + -o ${out} ${CMAKE_CURRENT_SOURCE_DIR}/test/${src} + DEPENDS test/${src}) + list(APPEND fixtures ${out}) + endforeach() + add_custom_target(search_fixtures ALL DEPENDS ${fixtures}) + + ida_add_idalib(test_search TYPE EXECUTABLE SOURCES test/test_search.cpp LIBRARIES PatternScanner::PatternScanner) + if(APPLE OR UNIX) + set_target_properties(test_search PROPERTIES BUILD_RPATH "${IDA_INSTALL_DIR}") + endif() + enable_testing() + add_test(NAME test_search COMMAND test_search ${fixtures}) + if(WIN32) + set_tests_properties(test_search PROPERTIES ENVIRONMENT "PATH=${IDA_INSTALL_DIR};$ENV{PATH}") + endif() +endif() diff --git a/plugins/ida/README.md b/plugins/ida/README.md new file mode 100644 index 0000000..8390af9 --- /dev/null +++ b/plugins/ida/README.md @@ -0,0 +1,46 @@ +# PatternScanner for IDA + +Search an IDA database with PatternScanner patterns, the way Search > Sequence of bytes works. IDA 9.x on Windows, +macOS and Linux. + +- **Shift-Alt-B** (also Search > PatternScanner pattern...): enter a byte pattern (`48 8B 05 X4 ? ? ? ? /d`) or + arm64 assembly (one instruction per line, `X` before the one to resolve, flags last). Options: + - **Find all occurrences**: lists every match with its result, segment, function and instruction. Enter or + double-click jumps to the match. + - **Search up**: the previous match instead of the next one. + - **From cursor**: start at the cursor rather than at the start (or end, searching up) of the database. +- **Ctrl-Shift-B**: repeat the last search from the cursor. + +A single search jumps to the match and prints the result (after X and /d /r) to the output window. + +/d, /r, `X` and /a follow the database's architecture, not the machine running IDA: byte patterns in an +arm64 database decode /d as arm64 instructions, and /a steps 4 bytes in 32-bit databases. Matches never span +segments or uninitialized bytes. + +## Build + +Needs the [IDA SDK](https://github.com/HexRaysSA/ida-sdk) with [ida-cmake](https://github.com/allthingsida/ida-cmake) +(neither is part of this repo): +- GitHub checkout of the SDK: `git submodule update --init src/cmake` +- Zip SDK: `git clone https://github.com/allthingsida/ida-cmake /ida-cmake` + +```sh +cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk +cmake --build build-ida --config Release +``` +The plugin (`pattern_scanner.dll` / `.so` / `.dylib`) is written to `$IDABIN/plugins`; IDABIN defaults to the SDK's +`bin` folder. Copy it to your IDA user plugins folder (`%APPDATA%\Hex-Rays\IDA Pro\plugins` on Windows, +`~/.idapro/plugins` on macOS and Linux), or pass `-DIDABIN=/path/to/ida` to build straight into IDA. + +The SDK ships link libraries for MSVC on Windows, clang on macOS and **gcc on Linux**: on Linux, configure with +`-DCMAKE_CXX_COMPILER=g++` if your default compiler is clang. + +## Test + +`test/test_search.cpp` runs the plugin's search (`search.hpp`) through idalib on small x86, x64 and arm64 ELF files +assembled from `test/*.S` with clang and lld. It needs an IDA install with idalib and a license: +```sh +cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk -DIDA_INSTALL_DIR=/path/to/ida +cmake --build build-ida --config Release +ctest --test-dir build-ida -C Release --output-on-failure +``` diff --git a/plugins/ida/pattern_scanner.cpp b/plugins/ida/pattern_scanner.cpp new file mode 100644 index 0000000..bce340f --- /dev/null +++ b/plugins/ida/pattern_scanner.cpp @@ -0,0 +1,206 @@ +// PatternScanner for IDA: search the database with PatternScanner byte or arm64 patterns, like Search > Sequence of +// bytes. Find the next/previous match from the cursor, or list every match with what /d /r resolve it to. +#include "search.hpp" +#include + +#include +#include +#include +#include +#include + +namespace { + // Find all: one row per match. Enter / double-click jumps to the match + struct hits_chooser_t : public chooser_t { + static constexpr int widths_[] = { CHCOL_HEX | 16, CHCOL_PLAIN | 32, 8, 24, 48 }; + static constexpr const char* const header_[] = { "Match", "Result", "Segment", "Function", "Instruction" }; + std::vector hits; + qstring title_; + + hits_chooser_t(qstring t, std::vector h) : + chooser_t(0, qnumber(widths_), widths_, header_), hits(std::move(h)), title_(std::move(t)) { + title = title_.c_str(); + } + size_t idaapi get_count() const override { return hits.size(); } + ea_t idaapi get_ea(size_t n) const override { return hits[n].match; } + cbret_t idaapi enter(size_t n) override { + jumpto(hits[n].match); + return cbret_t(); + } + void idaapi get_row(qstrvec_t* cols, int*, chooser_item_attrs_t*, size_t n) const override { + const auto& h = hits[n]; + (*cols)[0].sprnt("%" FMT_EA "X", h.match); + if (!h.resolved) + (*cols)[1] = "can't decode the instruction at X"; + else if (h.is_value) + (*cols)[1].sprnt("value 0x%llX", static_cast(h.result)); + else { + (*cols)[1].sprnt("%" FMT_EA "X", h.result); + qstring name = get_name(h.result); + if (!name.empty()) (*cols)[1].cat_sprnt(" %s", name.c_str()); + } + get_segm_name(&(*cols)[2], getseg(h.match)); + if (func_t* f = get_func(h.match)) get_func_name(&(*cols)[3], f->start_ea); + generate_disasm_line(&(*cols)[4], h.match, GENDSM_REMOVE_TAGS); + } + }; + + struct plugin_ctx_t; + + struct handler_t : public action_handler_t { + plugin_ctx_t& ctx; + bool repeat; + handler_t(plugin_ctx_t& c, bool r) : ctx(c), repeat(r) {} + int idaapi activate(action_activation_ctx_t*) override; + action_state_t idaapi update(action_update_ctx_t*) override { return AST_ENABLE_ALWAYS; } + }; + + constexpr const char search_action[] = "patternscanner:search"; + constexpr const char next_action[] = "patternscanner:next"; + + struct plugin_ctx_t : public plugmod_t { + handler_t search_handler{ *this, false }; + handler_t next_handler{ *this, true }; + // Dialog state, kept between searches + qstring text; + ushort kind = pscan::db_arch() == patterns::arch::arm64 ? pscan::arm64_pattern : pscan::bytes_pattern; + enum : ushort { opt_all = 1, opt_up = 2, opt_cursor = 4 }; + ushort opts = opt_cursor; + // The last pattern, for Search next + std::optional last; + + plugin_ctx_t() { + register_action(ACTION_DESC_LITERAL_PLUGMOD(search_action, "PatternScanner pattern...", &search_handler, + this, "Shift-Alt-B", "Search with a PatternScanner byte or arm64 pattern", -1)); + register_action(ACTION_DESC_LITERAL_PLUGMOD(next_action, "PatternScanner next", &next_handler, + this, "Ctrl-Shift-B", "Repeat the last PatternScanner search from the cursor", -1)); + attach_action_to_menu("Search/", search_action, SETMENU_APP); + attach_action_to_menu("Search/", next_action, SETMENU_APP); + } + + bool idaapi run(size_t) override { + search(); + return true; + } + + void search() { + static const char form[] = + "STARTITEM 0\n" + "BUTTON YES* Search\n" + "PatternScanner\n" + "\n" + "Bytes: 48 8B 05 X4 ? ? ? ? 48 85 C0 /d\n" + "arm64: one instruction per line, X before the one to resolve, flags last (e.g. /d)\n" + "<~P~attern:t:0:64:::>\n" + "<~B~ytes:R>\n" + "<~A~RM64 assembly:R>>\n" + "<~F~ind all occurrences:C>\n" + "\n" + ">\n"; + textctrl_info_t ti; + ti.text = text; + if (ask_form(form, &ti, &kind, &opts) != ASKBTN_YES) return; + text = ti.text; + // Parsed before listing unsupported lines: both throw on a line too long, and an exception escaping + // into IDA would take it down + std::optional parsed; + try { + parsed = pscan::parse(text, static_cast(kind)); + } + catch (const std::logic_error& e) { + last.reset(); + warning("PatternScanner: %s", e.what()); + return; + } + if (kind == pscan::arm64_pattern) { + const qstring unknown = pscan::unsupported_lines(text); + if (!unknown.empty() && ask_yn(ASKBTN_NO, "HIDECANCEL\nPatternScanner doesn't support these instructions, " + "so they would match any instruction:\n\n%s\nSearch anyway?", unknown.c_str()) != ASKBTN_YES) + return; + } + last = std::move(parsed); + const auto& p = pscan::base(*last); + if (!p.length()) { + last.reset(); + warning("PatternScanner: the pattern is empty"); + return; + } + if (opts & opt_all) { + show_wait_box("PatternScanner: searching..."); + auto hits = pscan::find_all(p); + hide_wait_box(); + if (hits.empty()) { + msg("PatternScanner: no matches\n"); + return; + } + qstring title; + title.sprnt("PatternScanner: %" FMT_Z " matches for %s", hits.size(), first_line().c_str()); + msg("%s\n", title.c_str()); + (new hits_chooser_t(title, std::move(hits)))->choose(); + return; + } + const bool up = opts & opt_up; + ea_t from = up ? inf_get_max_ea() : inf_get_min_ea(); + if (opts & opt_cursor) from = get_screen_ea(); + next(from); + } + + // Search next: from just past the cursor, in the last direction + void repeat() { + if (!last) { + search(); + return; + } + const ea_t here = get_screen_ea(); + next(opts & opt_up ? here : here + 1); + } + + void next(ea_t from) { + const bool up = opts & opt_up; + pscan::hit_t h; + show_wait_box("PatternScanner: searching..."); + const bool found = pscan::find_next(pscan::base(*last), from, up, &h); + hide_wait_box(); + if (!found) { + msg("PatternScanner: no match %s %" FMT_EA "X\n", up ? "before" : "from", from); + return; + } + jumpto(h.match); + if (!h.resolved) + msg("PatternScanner: match at %" FMT_EA "X, can't decode the instruction at X\n", h.match); + else if (h.is_value) + msg("PatternScanner: match at %" FMT_EA "X, value 0x%llX\n", h.match, static_cast(h.result)); + else + msg("PatternScanner: match at %" FMT_EA "X -> %" FMT_EA "X\n", h.match, h.result); + } + + qstring first_line() const { + qstring s = text; + const size_t nl = s.find('\n'); + if (nl != qstring::npos) s.resize(nl); + return s; + } + }; + + int idaapi handler_t::activate(action_activation_ctx_t*) { + if (repeat) ctx.repeat(); + else ctx.search(); + return 1; + } + + plugmod_t* idaapi init() { + return new plugin_ctx_t; + } +} + +plugin_t PLUGIN = { + IDP_INTERFACE_VERSION, + PLUGIN_MULTI, + init, + nullptr, + nullptr, + "Search the database with PatternScanner patterns", + "Shift-Alt-B: search (next/previous from the cursor, or find all)\nCtrl-Shift-B: search next", + "PatternScanner", + nullptr, +}; diff --git a/plugins/ida/search.hpp b/plugins/ida/search.hpp new file mode 100644 index 0000000..14da5a6 --- /dev/null +++ b/plugins/ida/search.hpp @@ -0,0 +1,123 @@ +#pragma once +// Database search with PatternScanner patterns. No UI, so the idalib test runs the same code as the plugin. +// The library headers go first: the SDK's pro.h redefines some C functions the standard headers use. +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace pscan { + enum kind_t : ushort { bytes_pattern, arm64_pattern }; + + // Architecture of the database being searched, not of the IDA running the plugin: decides /d /r, X and /a + inline patterns::arch db_arch() { + if (inf_is_64bit()) + return inf_get_procname() == "ARM" ? patterns::arch::arm64 : patterns::arch::x64; + return patterns::arch::x86; + } + + // Pattern has no virtual destructor (patterns aren't deleted through it), so the concrete type is kept + using pattern_t = std::variant; + + inline const patterns::Pattern& base(const pattern_t& p) { + return std::visit([](const auto& x) -> const patterns::Pattern& { return x; }, p); + } + + // Throws std::logic_error on invalid text + inline pattern_t parse(qstring text, kind_t kind) { + if (kind == arm64_pattern) + return patterns::RuntimeArmPattern(text.c_str(), text.length()); + // Byte patterns pasted over several lines + for (auto& c : text) + if (c == '\n' || c == '\r' || c == '\t') c = ' '; + return patterns::RuntimePattern(text.c_str(), text.length(), db_arch()); + } + + // arm64 lines the encoder doesn't know, one per line: they become wildcards matching any instruction. A line that + // is just "?" is a deliberate wildcard and isn't listed + inline qstring unsupported_lines(const qstring& text) { + qstring out; + patterns::detail::parse_arm(text.c_str(), text.length(), + [&](size_t, const char* instruction) { + if (!(instruction[0] == '?' && !instruction[1]) && !patterns::detail::a64_encode_instruction(instruction).mask) + out.cat_sprnt("%s\n", instruction); + }, + [](size_t) {}, [](const char*) {}); + return out; + } + + struct hit_t { + ea_t match = BADADDR; // where the pattern matched + ea_t result = BADADDR; // what find() gives: the X offset and /d /r applied + bool is_value = false; // result is a value read from the code (/r, arm64 /d on MOVZ/ADD/LDR offsets) + bool resolved = false; // false when /d couldn't decode the instruction at X + }; + + // Calls cb(hit) for every match starting in [start, end), in address order, until cb returns false. + // Matches don't span segments or uninitialized bytes. Returns false if the user cancelled. + template + bool for_each_match(const patterns::Pattern& p, ea_t start, ea_t end, F&& cb) { + segment_t* s = getseg(start); + if (!s) s = get_next_seg(start); + for (; s && s->start_ea < end; s = get_next_seg(s->start_ea)) { + if (user_cancelled()) return false; + const ea_t lo = s->start_ea > start ? s->start_ea : start; + const size_t n = s->end_ea - lo; + // The buffer sits at the same offset in a 4KB page as lo: /a and arm64 ADRP page math use its address + std::vector raw(n + 0x1000); + uint8_t* buf = raw.data() + ((lo - reinterpret_cast(raw.data())) & 0xFFF); + bytevec_t loaded; + loaded.resize((n + 7) / 8); + if (get_bytes(buf, n, lo, GMB_READALL, loaded.begin()) < 0) return false; + // Scan each run of initialized bytes on its own + for (size_t a = 0; a < n;) { + if (!loaded.test_bit(a)) { ++a; continue; } + size_t b = a; + while (b < n && loaded.test_bit(b)) ++b; + const uint8_t* run_end = buf + b; + for (const uint8_t* cur = buf + a; cur < run_end;) { + const uint8_t* m = p.match(cur, run_end - cur); + if (!m) break; + hit_t h; + h.match = lo + (m - buf); + if (h.match >= end) return true; + try { + const auto r = reinterpret_cast(p.resolve(m, run_end, &h.is_value)); + // Addresses come back in buffer terms; the arithmetic wraps, so targets outside it work too + h.result = h.is_value ? ea_t(r) : ea_t(lo + (r - reinterpret_cast(buf))); + if (!inf_is_64bit()) h.result &= 0xFFFFFFFF; + h.resolved = true; + } + catch (const std::logic_error&) {} + if (!cb(h)) return true; + cur = m + 1; + } + a = b; + } + } + return true; + } + + // Down: the first match at or after from. Up: the last match before from + inline bool find_next(const patterns::Pattern& p, ea_t from, bool up, hit_t* out) { + bool found = false; + if (up) + for_each_match(p, inf_get_min_ea(), from, [&](const hit_t& h) { *out = h; found = true; return true; }); + else + for_each_match(p, from, BADADDR, [&](const hit_t& h) { *out = h; found = true; return false; }); + return found; + } + + // Empty if cancelled + inline std::vector find_all(const patterns::Pattern& p) { + std::vector hits; + if (!for_each_match(p, inf_get_min_ea(), BADADDR, [&](const hit_t& h) { hits.push_back(h); return true; })) + hits.clear(); + return hits; + } +} diff --git a/plugins/ida/test/arm64.S b/plugins/ida/test/arm64.S new file mode 100644 index 0000000..2eb3768 --- /dev/null +++ b/plugins/ida/test/arm64.S @@ -0,0 +1,33 @@ +.text +.globl _start +_start: + ret +code_adrp: + movz w9, #0x7777 + adrp x0, target_data + add x0, x0, :lo12:target_data +code_bl: + movz w9, #0x6666 + bl target_func +target_func: + ret +// Two prologues: the first is a near miss for the STP/MOV pattern in test_search.cpp, the second matches it +near_miss: + sub sp, sp, #0x20 + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + mov x19, x1 + mov x0, x30 + ret +wanted: + stp x20, x19, [sp, #-0x20]! + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + mov x19, x1 + mov x0, x30 + ret + +.data +.balign 16 +.zero 0x2340 // target_data in a later page, at a page offset +target_data: .quad 0x1122334455667788 diff --git a/plugins/ida/test/test_search.cpp b/plugins/ida/test/test_search.cpp new file mode 100644 index 0000000..1d76fee --- /dev/null +++ b/plugins/ida/test/test_search.cpp @@ -0,0 +1,125 @@ +// Runs the plugin's search (search.hpp) through idalib on the fixture binaries. Exit code 0 = pass. +// Usage: test_search +#include "../search.hpp" +#include +#include + +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +static ea_t sym(const char* name) { + const ea_t ea = get_name_ea(BADADDR, name); + if (ea == BADADDR) { printf("FAIL: no symbol %s\n", name); exit(1); } + return ea; +} + +static pscan::hit_t next(const char* text, pscan::kind_t kind, ea_t from, bool up, bool* found = nullptr) { + const auto p = pscan::parse(text, kind); + pscan::hit_t h; + const bool f = pscan::find_next(pscan::base(p), from, up, &h); + if (found) *found = f; + else CHECK(f); + return h; +} + +static std::vector all(const char* text, pscan::kind_t kind = pscan::bytes_pattern) { + return pscan::find_all(pscan::base(pscan::parse(text, kind))); +} + +static void open(const char* path) { + if (open_database(path, true) != 0) { printf("FAIL: can't open %s\n", path); exit(1); } + auto_wait(); +} + +// x86 and x64: find next/previous, find all, /d /r, uninitialized bytes, /a by the database's pointer size +static void x86_family(const char* path, bool x64) { + open(path); + CHECK(pscan::db_arch() == (x64 ? patterns::arch::x64 : patterns::arch::x86)); + const ea_t m1 = sym("marker1"), m2 = sym("marker2"), m3 = sym("marker3"); + const char* marker = "DE AD BE EF 11 22"; + + const auto hits = all(marker); + CHECK(hits.size() == 3 && hits[0].match == m1 && hits[1].match == m2 && hits[2].match == m3); + CHECK(hits[0].resolved && !hits[0].is_value && hits[0].result == m1); + CHECK(next(marker, pscan::bytes_pattern, m1, false).match == m1); // down: from is included + CHECK(next(marker, pscan::bytes_pattern, m1 + 1, false).match == m2); + CHECK(next(marker, pscan::bytes_pattern, m2, true).match == m1); // up: before from + CHECK(next(marker, pscan::bytes_pattern, inf_get_max_ea(), true).match == m3); + bool found = true; + next(marker, pscan::bytes_pattern, m1, true, &found); + CHECK(!found); + next(marker, pscan::bytes_pattern, m3 + 1, false, &found); + CHECK(!found); + + const ea_t call = sym("code_call"), func = sym("target_func"); + const auto d = next("E8 X4 ? ? ? ? A3 B4 /d", pscan::bytes_pattern, 0, false); + CHECK(d.match == call && d.result == func && !d.is_value); + const auto r = next("E8 X4 ? ? ? ? A3 B4 /r", pscan::bytes_pattern, 0, false); + CHECK(r.is_value && r.result == ((func - (call + 5)) & (x64 ? ~0ull : 0xFFFFFFFFull))); + if (x64) { + const auto lea = next("48 8D 05 X4 ? ? ? ? CC CC A1 B2 /d", pscan::bytes_pattern, 0, false); + CHECK(lea.match == sym("code_lea") && lea.result == sym("target_data")); + } + + const ea_t zeros = sym("zeros"); + for (const auto& h : all("00 00 00 00 00 00 00 00")) + CHECK(h.match + 8 <= zeros || h.match >= zeros + 64); + + const auto aligned = all("C7 C8 C9 CA /a"); + CHECK(aligned.size() == (x64 ? 0u : 1u)); + CHECK(all("C7 C8 C9 CA").size() == 1); + + close_database(false); + printf(" %s ok\n", x64 ? "x64" : "x86"); +} + +// arm64: ADRP page math against database addresses, BL, values, byte patterns with arm64 semantics +static void arm64(const char* path) { + open(path); + CHECK(pscan::db_arch() == patterns::arch::arm64); + const ea_t data = sym("target_data"), func = sym("target_func"); + CHECK((data & 0xFFF) != 0); // the fixture puts it mid-page, so a page-misaligned buffer would show + + const auto adrl = next("MOVZ W9, #0x7777\nX ADRP X?, #?\nADD X?, X?, #?\n/d", pscan::arm64_pattern, 0, false); + CHECK(adrl.match == sym("code_adrp") && adrl.result == data && !adrl.is_value); + const auto bl = next("MOVZ W9, #0x6666\nX BL ?\n/d", pscan::arm64_pattern, 0, false); + CHECK(bl.match == sym("code_bl") && bl.result == func); + const auto movz = next("X MOVZ W9, #0x7777\n/d", pscan::arm64_pattern, 0, false); + CHECK(movz.is_value && movz.result == 0x7777); + // A byte pattern in an arm64 database: /d decodes the ADRP and follows the ADD + const auto bytes = next("E9 EE 8E 52 X ? ? ? ? /d", pscan::bytes_pattern, 0, false); + CHECK(bytes.match == sym("code_adrp") && bytes.result == data); + CHECK(all("NOP\nRET", pscan::arm64_pattern).empty()); + + // A prologue as IDA prints it (STP, MOV): the first hit must be the function that has it, not a near miss + const char* prologue = "STP X20, X19, [SP,#?]!\n" + "STP X29, X30, [SP,#?]\n" + "ADD X29, SP, #0x10\n" + "MOV X19, X1\n" + "MOV X0, X30"; + CHECK(pscan::unsupported_lines(prologue).empty()); + const auto fn = all(prologue, pscan::arm64_pattern); + CHECK(fn.size() == 1 && fn[0].match == sym("wanted")); + + // Unknown instructions are reported (they'd match anything); a deliberate "?" line isn't + CHECK(pscan::unsupported_lines("NOP\nFOO X1, X2\n ? \nstp x29, x30, [sp,#?]\nBAR\n/d") == "FOO X1, X2\nBAR\n"); + + bool threw = false; + try { pscan::parse("48 8G", pscan::bytes_pattern); } catch (const std::logic_error&) { threw = true; } + CHECK(threw); + + close_database(false); + puts(" arm64 ok"); +} + +int main(int argc, char** argv) { + if (argc != 4) { puts("usage: test_search "); return 2; } + if (init_library() != 0) { puts("FAIL: init_library (idalib missing or no license?)"); return 1; } + x86_family(argv[1], true); + x86_family(argv[2], false); + arm64(argv[3]); + puts("all search tests passed"); +} diff --git a/plugins/ida/test/x86.S b/plugins/ida/test/x86.S new file mode 100644 index 0000000..9b2a78d --- /dev/null +++ b/plugins/ida/test/x86.S @@ -0,0 +1,30 @@ +// Shared by the x86 and x64 fixtures (see CMakeLists.txt): same layout, different databases +.intel_syntax noprefix +.text +.globl _start +_start: + ret +marker1: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +code_call: + call target_func + .byte 0xA3, 0xB4 +marker2: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +.balign 8 + .byte 0x90, 0x90, 0x90, 0x90 +unaligned8: .byte 0xC7, 0xC8, 0xC9, 0xCA // 4 mod 8: found by /a in x86 databases only +#ifdef __x86_64__ +code_lea: + lea rax, [rip + target_data] + .byte 0xCC, 0xCC, 0xA1, 0xB2 +#endif +marker3: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +target_func: + ret + +.data +.balign 16 +target_data: .quad 0x1122334455667788 + +.bss +.balign 16 +zeros: .zero 64 // uninitialized: never matched diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..348b9db --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,76 @@ +# Tests and benchmarks. Not part of the library; the headers stay include-only. +# cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release +# cmake --build build --config Release +# ctest --test-dir build -C Release --output-on-failure +cmake_minimum_required(VERSION 3.16) +project(PatternScannerTests CXX) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release) +endif() +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +find_package(Threads REQUIRED) +add_subdirectory(.. PatternScanner) +link_libraries(PatternScanner::PatternScanner) +add_compile_definitions(_CRT_SECURE_NO_WARNINGS) # fopen in the test code +enable_testing() + +add_executable(test_patterns test_patterns.cpp) +add_executable(test_patterns_nosimd test_patterns.cpp) +target_compile_definitions(test_patterns_nosimd PRIVATE PATTERNS_NO_SIMD) +add_executable(test_cpp17 test_cpp17.cpp) +add_executable(test_arm test_arm.cpp) +set_target_properties(test_cpp17 PROPERTIES CXX_STANDARD 17) +add_executable(example_usage example_usage.cpp) +add_executable(bench_scan bench_scan.cpp) +add_executable(bench_multi bench_multi.cpp) + +foreach(t test_patterns test_patterns_nosimd bench_multi) + target_link_libraries(${t} PRIVATE Threads::Threads) +endforeach() + +add_test(NAME test_patterns COMMAND test_patterns) +add_test(NAME test_patterns_nosimd COMMAND test_patterns_nosimd) +add_test(NAME test_cpp17 COMMAND test_cpp17) +add_test(NAME test_arm COMMAND test_arm) +add_test(NAME example_usage COMMAND example_usage) +if(WIN32) + add_executable(test_tls test_tls.cpp) + add_test(NAME test_tls COMMAND test_tls) +endif() + +# No pattern text / XOR plaintext left in the binary, at every optimization level and with LTO (whole-program +# inlining folds differently), with debug info, C++17 (macros) and C++20 (macros and literals) +include(CheckIPOSupported) +check_ipo_supported(RESULT lto_supported LANGUAGES CXX) +if(MSVC) + set(artifact_opts Od O1 O2) + set(artifact_lto_opts O2) + set(artifact_flag /) + set(artifact_debug /Zi) +else() + set(artifact_opts O0 O1 O2 O3 Os) + set(artifact_lto_opts O2 O3) + set(artifact_flag -) + set(artifact_debug -g) +endif() +if(lto_supported) + foreach(opt ${artifact_lto_opts}) + list(APPEND artifact_opts ${opt}_lto) + endforeach() +endif() +foreach(std 17 20) + foreach(opt ${artifact_opts}) + set(t test_artifacts_cpp${std}_${opt}) + string(REPLACE "_lto" "" level ${opt}) + add_executable(${t} test_artifacts.cpp) + set_target_properties(${t} PROPERTIES CXX_STANDARD ${std}) + if(NOT level STREQUAL opt) + set_target_properties(${t} PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON) + endif() + # After the build type's flags, so this level wins + target_compile_options(${t} PRIVATE ${artifact_flag}${level} ${artifact_debug}) + add_test(NAME ${t} COMMAND ${t}) + endforeach() +endforeach() diff --git a/tests/bench_multi.cpp b/tests/bench_multi.cpp new file mode 100644 index 0000000..5c91aa1 --- /dev/null +++ b/tests/bench_multi.cpp @@ -0,0 +1,67 @@ +// Resolving many patterns over one module: sequential vs parallel per pattern vs cache-chunked. +// bench_multi synthetic 96 MB module with 60 planted signatures +// bench_multi 60 unique signatures sampled from that file +#include +#include "multi_scan.hpp" +#include +#include +#include +#include +#include + +template static double ms(F f) { + const auto t = std::chrono::steady_clock::now(); + f(); + return std::chrono::duration(std::chrono::steady_clock::now() - t).count(); +} + +static std::vector load(const char* path) { + std::vector v; + if (FILE* f = fopen(path, "rb")) { + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) v.insert(v.end(), chunk, chunk + n); + fclose(f); + } + return v; +} + +int main(int argc, char** argv) { + std::mt19937 rng(5); + std::vector mod; + if (argc > 1) { + mod = load(argv[1]); + if (mod.size() < 1 << 16) { printf("can't read %s\n", argv[1]); return 1; } + } else { + mod.resize(96 << 20); + for (auto& b : mod) b = static_cast(rng()); + } + + // Signatures from the back 3/4 of the module, kept only if unique, so every scan is a long one + std::vector owned; + while (owned.size() < 60) { + const size_t off = mod.size() / 4 + rng() % (mod.size() * 3 / 4 - 64); + std::string s; + char hex[4]; + for (int k = 0; k < 16; ++k) { + if (k >= 3 && k <= 6) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", mod[off + k]); s += hex; } + } + patterns::RuntimePattern p(s.c_str()); + if (p.find(mod.data(), mod.size()) == mod.data() + off) owned.push_back(std::move(p)); + } + std::vector pats; + for (auto& p : owned) pats.push_back(&p); + + const unsigned threads = std::thread::hardware_concurrency(); + printf("%zu signatures, %.0f MB module, %u hardware threads\n", pats.size(), mod.size() / 1048576.0, threads); + std::vector want, got; + const auto run = [&](const char* name, auto&& f) { + double best = 1e9; + for (int r = 0; r < 3; ++r) best = std::min(best, ms([&] { f(); })); + printf(" %-34s %8.1f ms %s\n", name, best, got == want ? "" : "RESULTS DIFFER"); + }; + run("sequential (find() per pattern)", [&] { multi::sequential(mod.data(), mod.size(), pats, want); got = want; }); + run("patterns::find_all (1 thread)", [&] { got.resize(pats.size()); patterns::find_all(mod.data(), mod.size(), pats.data(), pats.size(), got.data()); }); + run("parallel per pattern", [&] { multi::per_pattern(mod.data(), mod.size(), pats, got, threads); }); + run("chunked, all threads", [&] { multi::chunked(mod.data(), mod.size(), pats, got, threads); }); +} diff --git a/tests/bench_scan.cpp b/tests/bench_scan.cpp new file mode 100644 index 0000000..bb042ea --- /dev/null +++ b/tests/bench_scan.cpp @@ -0,0 +1,66 @@ +// Single-pattern scan speed. +// bench_scan synthetic 64 MB of random bytes +// bench_scan also 40 real signatures sampled from that file (e.g. a game exe or dylib) +#include +#include +#include +#include +#include +#include + +template static double ms(F f) { + const auto t = std::chrono::steady_clock::now(); + f(); + return std::chrono::duration(std::chrono::steady_clock::now() - t).count(); +} + +static std::vector load(const char* path) { + std::vector v; + if (FILE* f = fopen(path, "rb")) { + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) v.insert(v.end(), chunk, chunk + n); + fclose(f); + } + return v; +} + +// 16 bytes from the module with a rel32 wildcarded, like "48 8B 05 ? ? ? ? 48 85 C0 ..." +static std::string signature_at(const std::vector& d, size_t off) { + std::string s; + char hex[4]; + for (int k = 0; k < 16; ++k) { + if (k >= 3 && k <= 6) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", d[off + k]); s += hex; } + } + return s; +} + +int main(int argc, char** argv) { + { + std::vector buf(64 << 20); + std::mt19937 rng(1); + for (auto& b : buf) b = static_cast(rng()); + const uint8_t sig[] = { 0x48, 0x8B, 0x05, 0xAA, 0xBB, 0xCC, 0xDD, 0x48, 0x85, 0xC0, 0x74, 0x10 }; + memcpy(&buf[buf.size() - 4096], sig, sizeof sig); + auto p = "48 8B 05 ? ? ? ? 48 85 C0 74 10"_rtpattern; + double best = 1e9; + for (int r = 0; r < 5; ++r) best = std::min(best, ms([&] { p.find(buf.data(), buf.size()); })); + printf("synthetic: 64 MB random, match at end %7.2f ms (%.1f GB/s)\n", best, 64 / 1024.0 / (best / 1000)); + } + if (argc > 1) { + const auto d = load(argv[1]); + if (d.size() < 1024) { printf("can't read %s\n", argv[1]); return 1; } + std::mt19937 rng(42); + double total = 0; + size_t scanned = 0; + for (int s = 0; s < 40; ++s) { + const size_t off = rng() % (d.size() - 64); + patterns::RuntimePattern p(signature_at(d, off).c_str()); + const uint8_t* hit = nullptr; + total += ms([&] { hit = p.find(d.data(), d.size()); }); + scanned += hit - d.data() + 16; // signatures can match earlier than where they were sampled + } + printf("real: 40 signatures from %s (%.1f MB) %7.2f ms (%.1f GB/s)\n", argv[1], d.size() / 1048576.0, total, + scanned / 1073741824.0 / (total / 1000)); + } +} diff --git a/tests/example_usage.cpp b/tests/example_usage.cpp new file mode 100644 index 0000000..8eb7cc2 --- /dev/null +++ b/tests/example_usage.cpp @@ -0,0 +1,153 @@ +// Every way to fill pattern-resolved globals, against a synthetic "module" that must be decrypted before it can be +// scanned. Each result is logged as [count] name address (address rebased to the disassembler's image base). +// +// 1. Per global, at static init: inline T name = scan("name", "..."_ctpattern); +// Simplest. One full pass over the module per global. +// 2. Batched, at static init: one list; the first of its globals to initialize runs find_all for all of them. +// One pass for the whole list. +// 3. Explicit call: same list; resolve_pointers(start, size) whenever the module is ready (main, a TLS +// callback, DllMain, after decryption...). test_tls.cpp runs find_all from a real TLS callback. +// +// The module and anything that must happen before scanning (decryption) live in game_module(), which runs once on +// first use, from whichever initializer gets there first; the initialization order of globals across files is +// unspecified, so a plain global holding the range could still be unset. +// In a DLL, static init, TLS callbacks and DllMain run under the loader lock: don't wait on other threads there. +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#define BASEADDR 0x100000000 +#else +#define BASEADDR 0x140000000 +#endif + +// ---- the module ------------------------------------------------------------------------------------------------ +// Where the code sequences the patterns look for are planted +constexpr size_t at_one = 0x012340, at_two = 0x0ABCD0, at_four = 0x5F0010, at_five = 0xC00050; +constexpr uint8_t module_key = 0x5A; + +static std::vector load_encrypted_module() { + std::vector m(16 << 20); + std::mt19937 rng(9); + for (auto& b : m) b = static_cast(rng()); + auto plant = [&](size_t at, std::initializer_list bytes) { std::copy(bytes.begin(), bytes.end(), m.begin() + at); }; + plant(at_one, { 0x48, 0x8B, 0x05, 0x11, 0x22, 0x33, 0x44, 0x48, 0x85, 0xC0, 0x74, 0x05, 0xE8 }); + plant(at_two, { 0x40, 0x53, 0x48, 0x83, 0xEC, 0x20, 0x8B, 0x0D, 1, 2, 3, 4, 0x85, 0xC9, 0x0F, 0x84 }); + plant(at_four, { 0x4C, 0x8D, 0x05, 9, 8, 7, 6, 0x48, 0x8B, 0xD3, 0x48, 0x8B, 0xCF, 0xE8 }); + plant(at_five, { 0xF3, 0x0F, 0x10, 0x05, 5, 6, 7, 8, 0x0F, 0x2F, 0xC1, 0x76 }); + for (auto& b : m) b ^= module_key; + return m; +} + +// Every scan gets its range from here. A real one would find the module (GetModuleHandle, dlopen, dyld...) and +// decrypt it or wait until it's decrypted +inline std::pair game_module() { + static const std::vector image = [] { + auto m = load_encrypted_module(); + for (auto& b : m) b ^= module_key; + return m; + }(); + return { image.data(), image.size() }; +} + +// ---- logging --------------------------------------------------------------------------------------------------- +inline std::atomic g_scan_count{0}, g_found_count{0}; + +inline void log_result(const char* name, const void* p) { + const int n = ++g_scan_count; + if (!p) { + printf("[%2d]\t%40s\tnot found\n", n, name); + return; + } + ++g_found_count; + const auto rebased = reinterpret_cast(p) - reinterpret_cast(game_module().first) + BASEADDR; + printf("[%2d]\t%40s\t%p (%p)\n", n, name, p, reinterpret_cast(rebased)); +} + +// ---- 1. per global, at static init ----------------------------------------------------------------------------- +// Takes the exact pattern type: MSVC rejects a compile-time-only derived temporary bound to const Pattern& (C7595) +template T scan(const char* name, const P& pattern) { + const auto [start, size] = game_module(); + const T result = pattern.template find(start, size); + log_result(name, reinterpret_cast(result)); + return result; +} + +inline void* g_one = scan("g_one", "48 8B 05 X ? ? ? ? 48 85 C0 74 ? E8"_ctpattern); +inline uint32_t* g_two = scan("g_two", "40 53 48 83 EC 20 8B 0D X ? ? ? ? 85 C9 0F 84"_xorpattern); +inline void* g_three = scan("g_three", "DE AD BE EF 13 37 C0 DE 00 11"_ctpattern); + +// ---- one list for 2 and 3 -------------------------------------------------------------------------------------- +#define POINTERS(X) \ + X(void*, g_four, "4C 8D 05 X ? ? ? ? 48 8B D3 48 8B CF E8") \ + X(float*, g_five, "F3 0F 10 05 X ? ? ? ? 0F 2F C1 76") \ + X(void*, g_missing, "DE AD BE EF 13 37 C0 DE 00 11 22") + +// All built at compile time (no initializer code), so usable before static init too, e.g. from a TLS callback +#define PATTERN_OF(type, name, sig) inline constexpr auto name##_pattern = sig##_ctpattern; +POINTERS(PATTERN_OF) +#define INDEX_OF(type, name, sig) name##_index, +enum : size_t { POINTERS(INDEX_OF) pointer_count }; +#define ADDRESS_OF(type, name, sig) &name##_pattern, +inline constexpr const patterns::Pattern* pointer_patterns[] = { POINTERS(ADDRESS_OF) }; +#define NAME_OF(type, name, sig) #name, // drop in release builds if the names shouldn't be in the binary +inline constexpr const char* pointer_names[] = { POINTERS(NAME_OF) }; + +// ---- 2. batched, at static init -------------------------------------------------------------------------------- +inline void* const* resolved() { + static const auto out = [] { + std::array r{}; + const auto [start, size] = game_module(); + patterns::find_all(start, size, pointer_patterns, pointer_count, r.data()); + for (size_t i = 0; i < pointer_count; ++i) log_result(pointer_names[i], r[i]); + return r; + }(); + return out.data(); +} +#define BATCHED_GLOBAL(type, name, sig) inline type name = static_cast(resolved()[name##_index]); +POINTERS(BATCHED_GLOBAL) + +// ---- 3. explicit call ------------------------------------------------------------------------------------------ +#define LATE_GLOBAL(type, name, sig) inline type late_##name = nullptr; +POINTERS(LATE_GLOBAL) +#define SETTER_OF(type, name, sig) [](void* p) { late_##name = static_cast(p); }, +inline constexpr void (*pointer_setters[])(void*) = { POINTERS(SETTER_OF) }; + +// No allocation or threads: safe under the loader lock. Returns how many resolved +inline size_t resolve_pointers(const uint8_t* start, size_t size) { + void* out[pointer_count]; + const size_t found = patterns::find_all(start, size, pointer_patterns, pointer_count, out); + for (size_t i = 0; i < pointer_count; ++i) { + pointer_setters[i](out[i]); + log_result(pointer_names[i], out[i]); + } + return found; +} + +// Proves the list needs no runtime initialization +constinit const patterns::Pattern* const* patterns_are_constant = pointer_patterns; +constinit void (* const* setters_are_constant)(void*) = pointer_setters; + +int main() { + puts("-- main: explicit resolve_pointers --"); + const auto [base, size] = game_module(); + const size_t late_found = resolve_pointers(base, size); + printf("%d of %d found\n", g_found_count.load(), g_scan_count.load()); + + const bool ok = g_one == base + at_one + 3 + && g_two == reinterpret_cast(base + at_two + 8) + && !g_three + && g_four == base + at_four + 3 + && g_five == reinterpret_cast(base + at_five + 4) + && !g_missing + && late_found == 2 && late_g_four == g_four && late_g_five == g_five && !late_g_missing + && g_scan_count == 9 && g_found_count == 6; + puts(ok ? "usage ok" : "FAIL usage"); + return !ok; +} diff --git a/tests/multi_scan.hpp b/tests/multi_scan.hpp new file mode 100644 index 0000000..6da5c0b --- /dev/null +++ b/tests/multi_scan.hpp @@ -0,0 +1,70 @@ +#pragma once +// Strategies for resolving many patterns over one module, for comparison with patterns::find_all. +// Test/benchmark code, not part of the library. Built only on the public Pattern API (match / resolve). +#include +#include +#include +#include +#include + +namespace multi { + // One full scan per pattern, one after another (what a list of find() calls does) + inline void sequential(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out) { + out.assign(pats.size(), nullptr); + for (size_t i = 0; i < pats.size(); ++i) + out[i] = pats[i]->find(start, size); + } + + // One full scan per pattern, patterns spread over threads + inline void per_pattern(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out, + unsigned threads = std::thread::hardware_concurrency()) { + out.assign(pats.size(), nullptr); + std::atomic next{0}; + std::vector pool; + for (unsigned t = 0; t < std::max(1u, threads); ++t) + pool.emplace_back([&] { + for (size_t i; (i = next++) < pats.size();) + out[i] = pats[i]->find(start, size); + }); + for (auto& t : pool) t.join(); + } + + // Module split into cache-sized chunks; every pattern runs over a chunk while it's hot in cache, so the module + // is read from RAM once instead of once per pattern. Chunks are spread over threads; threads = 1 runs inline. + inline void chunked(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out, + unsigned threads = std::thread::hardware_concurrency(), size_t chunk = 1 << 20) { + size_t overlap = 0; + for (auto p : pats) overlap = std::max(overlap, p->length()); + // Lowest match per pattern; chunks finish out of order, so keep the minimum rather than the first found + std::vector> best(pats.size()); + for (auto& b : best) b = UINTPTR_MAX; + const size_t chunks = (size + chunk - 1) / chunk; + std::atomic next{0}; + auto work = [&] { + for (size_t c; (c = next++) < chunks;) { + const uint8_t* s = start + c * chunk; + // Overlap into the next chunk so matches straddling the boundary are found + const size_t n = std::min(chunk + overlap - 1, size - c * chunk); + for (size_t i = 0; i < pats.size(); ++i) { + if (best[i].load(std::memory_order_relaxed) < reinterpret_cast(s)) + continue; // an earlier chunk already matched + if (auto hit = pats[i]->match(s, n)) { + uintptr_t v = reinterpret_cast(hit), cur = best[i]; + while (v < cur && !best[i].compare_exchange_weak(cur, v)) {} + } + } + } + }; + if (threads <= 1) { + work(); + } else { + std::vector pool; + for (unsigned t = 0; t < threads; ++t) pool.emplace_back(work); + for (auto& t : pool) t.join(); + } + out.assign(pats.size(), nullptr); + for (size_t i = 0; i < pats.size(); ++i) + if (best[i] != UINTPTR_MAX) + out[i] = pats[i]->resolve(reinterpret_cast(best[i].load()), start + size); + } +} diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp new file mode 100644 index 0000000..82a9916 --- /dev/null +++ b/tests/test_arm.cpp @@ -0,0 +1,561 @@ +// ArmPattern tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. +// The encoder and /d are checked on every platform against encodings produced by clang's assembler. On Apple arm64 +// the test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +// Fully specified instruction -> its encoding. The pattern must be exactly the encoding, every bit masked +#define CONCRETE(X) \ + X("NOP", 0xD503201F) \ + X("STP X20, X19, [SP,#-0x20]!", 0xA9BE4FF4) \ + X("STP X29, X30, [SP,#0x10]", 0xA9017BFD) \ + X("STP X29, X30, [SP],#0x10", 0xA8817BFD) \ + X("STP W1, W2, [X0,#8]", 0x29010801) \ + X("LDP X29, X30, [SP,#0x10]", 0xA9417BFD) \ + X("MOV X19, X1", 0xAA0103F3) \ + X("MOV X0, X30", 0xAA1E03E0) \ + X("MOV W0, W1", 0x2A0103E0) \ + X("MOV X29, SP", 0x910003FD) \ + X("MOV SP, X29", 0x910003BF) \ + X("MOV W0, #0x10", 0x52800200) \ + X("MOV X0, #0x10000", 0xD2A00020) \ + X("MOV W0, #-1", 0x12800000) \ + X("MOV X0, #-2", 0x92800020) \ + X("MOV W0, #0x10001", 0x320083E0) \ + X("MOV X0, #0xFFFF0000FFFF0000", 0xB2103FE0) \ + X("RET", 0xD65F03C0) \ + X("BR X16", 0xD61F0200) \ + X("BLR X8", 0xD63F0100) \ + X("B #0x100", 0x14000040) \ + X("B #-0x8", 0x17FFFFFE) \ + X("BL #0x1000", 0x94000400) \ + X("BL #-0x40", 0x97FFFFF0) \ + X("ADR X0, #0x10", 0x10000080) \ + X("ADR X3, #-0x4", 0x10FFFFE3) \ + X("ADR X5, #0x5", 0x30000025) \ + X("ADRP X0, #0x1000", 0xB0000000) \ + X("ADRP X8, #0x7000", 0xF0000028) \ + X("ADRP X1, #-0x2000", 0xD0FFFFE1) \ + X("ADRP X2, #0x123000", 0xF0000902) \ + X("LDRH W0, [X1,#0x10]", 0x79402020) \ + X("LDRH W0, [X1]", 0x79400020) \ + X("LDRH W2, [X3,#2]!", 0x78402C62) \ + X("LDRH W2, [X3],#2", 0x78402462) \ + X("LDRH W4, [X5,X6]", 0x786668A4) \ + X("LDRH W4, [X5,X6,LSL#1]", 0x786678A4) \ + X("LDR W0, [X1,#0x18]", 0xB9401820) \ + X("LDR X0, [X1,#0x18]", 0xF9400C20) \ + X("LDR X0, [X1]", 0xF9400020) \ + X("LDR W9, [SP,#0x24]", 0xB94027E9) \ + X("LDR X0, [X1,#8]!", 0xF8408C20) \ + X("LDR X0, [X1],#8", 0xF8408420) \ + X("LDR W0, [X1,#-4]!", 0xB85FCC20) \ + X("LDR X2, [X3,X4]", 0xF8646862) \ + X("LDR X2, [X3,X4,LSL#3]", 0xF8647862) \ + X("LDR W2, [X3,W4,UXTW#2]", 0xB8645862) \ + X("LDR X2, [X3,W4,SXTW]", 0xF864C862) \ + X("LDR X0, #0x100", 0x58000800) \ + X("LDR W1, #-0x10", 0x18FFFF81) \ + X("MOVZ W0, #0x1234", 0x52824680) \ + X("MOVZ X0, #0x1234, LSL#16", 0xD2A24680) \ + X("MOVZ X3, #0xFFFF, LSL#48", 0xD2FFFFE3) \ + X("MOVN W1, #0", 0x12800001) \ + X("MOVK X2, #0xBEEF, LSL#32", 0xF2D7DDE2) \ + X("ORR X0, X1, #0xFF", 0xB2401C20) \ + X("ORR W0, W1, #0x1", 0x32000020) \ + X("ORR X0, X1, #0xFFFF0000FFFF0000", 0xB2103C20) \ + X("ORR X0, X1, X2", 0xAA020020) \ + X("ORR W0, W1, W2, LSL#4", 0x2A021020) \ + X("ORR X0, X1, X2, ROR#7", 0xAAC21C20) \ + X("LDP X29, X30, [SP,#0x10]", 0xA9417BFD) \ + X("LDP W0, W1, [X2,#8]", 0x29410440) \ + X("LDP X29, X30, [SP],#0x20", 0xA8C27BFD) \ + X("LDP X0, X1, [X2,#-0x10]!", 0xA9FF0440) \ + X("ADD X0, X1, #0x10", 0x91004020) \ + X("ADD W0, W1, #0x10", 0x11004020) \ + X("ADD X0, X1, #1, LSL#12", 0x91400420) \ + X("ADD X0, SP, #0x20", 0x910083E0) \ + X("ADD X0, X1, X2", 0x8B020020) \ + X("ADD X0, X1, X2, LSL#3", 0x8B020C20) \ + X("ADD X0, X1, W2, UXTW", 0x8B224020) \ + X("ADD X0, X1, W2, SXTW#2", 0x8B22C820) \ + X("ADDS X0, X1, #4", 0xB1001020) \ + X("SUB SP, SP, #0x40", 0xD10103FF) \ + X("SUB X0, X1, X2", 0xCB020020) \ + X("SUBS W0, W1, #1", 0x71000420) \ + X("STR W0, [SP,#0x1C]", 0xB9001FE0) \ + X("STR X0, [X1,#0x18]", 0xF9000C20) \ + X("STR X0, [X1]", 0xF9000020) \ + X("STR X0, [SP,#-0x10]!", 0xF81F0FE0) \ + X("STR X0, [X1],#8", 0xF8008420) \ + X("STR W0, [X1,X2,LSL#2]", 0xB8227820) \ + X("CBZ X0, #0x40", 0xB4000200) \ + X("CBZ W0, #-0x8", 0x34FFFFC0) \ + X("CBNZ X1, #0x100", 0xB5000801) \ + X("LDR X0, [X1,#4]", 0xF8404020) \ + X("LDR X0, [X29,#-8]", 0xF85F83A0) \ + X("LDR W0, [X1,#-4]", 0xB85FC020) \ + X("STR X0, [X1,#-8]", 0xF81F8020) \ + X("STR WZR, [SP,#0xC]", 0xB9000FFF) \ + X("LDRH W0, [X1,#3]", 0x78403020) \ + X("LDUR X0, [X1,#0x10]", 0xF8410020) \ + X("STUR W0, [X29,#-4]", 0xB81FC3A0) \ + X("LDURB W0, [X1,#-1]", 0x385FF020) \ + X("STURH W0, [X1,#-2]", 0x781FE020) \ + X("LDURSW X0, [X1,#-4]", 0xB89FC020) \ + X("LDRB W0, [X1,#0x10]", 0x39404020) \ + X("LDRB W0, [X1,#3]", 0x39400C20) \ + X("LDRB W0, [X1,#1]!", 0x38401C20) \ + X("LDRB W0, [X1],#1", 0x38401420) \ + X("LDRB W0, [X1,X2]", 0x38626820) \ + X("LDRB W0, [X1,X2,LSL#0]", 0x38627820) \ + X("LDRB W0, [X1,W2,UXTW]", 0x38624820) \ + X("STRB W0, [X1,#0x10]", 0x39004020) \ + X("STRB WZR, [X1]", 0x3900003F) \ + X("STRH W0, [X1,#0x10]", 0x79002020) \ + X("STRH W0, [X1,#-2]!", 0x781FEC20) \ + X("LDRSW X0, [X1,#0x10]", 0xB9801020) \ + X("LDRSW X0, [X1,X2,LSL#2]", 0xB8A27820) \ + X("CMP X0, #0x10", 0xF100401F) \ + X("CMP W0, W1", 0x6B01001F) \ + X("CMP X0, X1, LSL#2", 0xEB01081F) \ + X("CMP X0, W1, UXTW", 0xEB21401F) \ + X("CMP W0, #1, LSL#12", 0x7140041F) \ + X("CMN W0, #1", 0x3100041F) \ + X("CMN X0, X1", 0xAB01001F) \ + X("TST W0, #0xFF", 0x72001C1F) \ + X("TST X0, X1", 0xEA01001F) \ + X("TST X0, #0x8000000000000000", 0xF241001F) \ + X("AND X0, X1, #0xFF", 0x92401C20) \ + X("AND W0, W1, W2", 0x0A020020) \ + X("ANDS X0, X1, X2, LSR#3", 0xEA420C20) \ + X("EOR W0, W1, #0x80000000", 0x52010020) \ + X("EOR X0, X1, X2", 0xCA020020) \ + X("B.EQ #0x10", 0x54000080) \ + X("B.NE #-0x8", 0x54FFFFC1) \ + X("B.HS #0x100", 0x54000802) \ + X("B.CS #0x100", 0x54000802) \ + X("B.LO #0x100", 0x54000803) \ + X("B.GT #0x20", 0x5400010C) \ + X("RETAA", 0xD65F0BFF) \ + X("RETAB", 0xD65F0FFF) \ + X("PACIASP", 0xD503233F) \ + X("PACIBSP", 0xD503237F) \ + X("AUTIASP", 0xD50323BF) \ + X("AUTIBSP", 0xD50323FF) \ + X("ORR X0, X1, X2, LSL #3", 0xAA020C20) \ + X("TST X0, X1, LSL #2", 0xEA01081F) \ + X("ORR X0, X1, X2, ROR #5", 0xAAC21420) \ + X("ADD X0, X1, #0x1000", 0x91400420) \ + X("SUB SP, SP, #0x1000", 0xD14007FF) \ + X("ADD X0, X1, #-8", 0xD1002020) \ + X("CMP X0, #-1", 0xB100041F) \ + X("ADD X0, SP, X1", 0x8B2163E0) \ + X("ADD SP, SP, X1", 0x8B2163FF) \ + X("ADD X0, SP, X1, LSL #2", 0x8B216BE0) \ + X("ADD W0, WSP, W1", 0x0B2143E0) \ + X("LDP X29, X30, [SP]", 0xA9407BFD) \ + X("STP X29, X30, [SP]", 0xA9007BFD) \ + X("ADR X0, #-1", 0x70FFFFE0) + +// Wildcard pattern, an instruction, its encoding, and whether the pattern must match it +#define WILDCARD(X) \ + X("STP X29, X30, [SP,#?]", "STP X29, X30, [SP,#0x30]", 0xA9037BFD, 1) \ + X("STP X29, X30, [SP,#?]", "LDP X29, X30, [SP,#0x10]", 0xA9417BFD, 0) \ + X("STP X29, X30, [SP,#?]", "STP X29, X30, [SP],#0x10", 0xA8817BFD, 0) \ + X("STP X20, X19, [SP,#?]!", "STP X20, X19, [SP,#-0x20]!", 0xA9BE4FF4, 1) \ + X("STP X20, X19, [SP,#?]!", "SUB SP, SP, #0x20", 0xD10083FF, 0) \ + X("MOV X?, X1", "MOV X5, X1", 0xAA0103E5, 1) \ + X("MOV X19, X?", "MOV X19, X1", 0xAA0103F3, 1) \ + X("MOV W0, #?", "MOV W0, #5", 0x528000A0, 1) \ + X("MOV W0, #?", "MOV W0, #-1", 0x12800000, 1) \ + X("MOV W0, #?", "MOV W0, W1", 0x2A0103E0, 0) \ + X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x18]", 0xB9401923, 1) \ + X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x1C]", 0xB9401D23, 0) \ + X("LDR W?, [X?,#0x18]", "LDR X3, [X9,#0x18]", 0xF9400D23, 0) \ + X("LDR W?, [X?,#0x18]", "STR W3, [X9,#0x18]", 0xB9001923, 0) \ + X("LDR ?, [X?,#?]", "LDR X3, [X9,#0x18]", 0xF9400D23, 1) \ + X("LDR ?, [X?,#?]", "LDR W3, [SP,#0x40]", 0xB94043E3, 1) \ + X("LDR X?, [X?,#?]", "LDR X0, [X1,#8]!", 0xF8408C20, 0) \ + X("LDR X?, [X?],#?", "LDR X0, [X1],#8", 0xF8408420, 1) \ + X("LDR X?, [X?,#?]!", "LDR X0, [X1,#-8]!", 0xF85F8C20, 1) \ + X("LDR X?, [X?,X?,LSL#3]", "LDR X0, [X1,X2,LSL#3]", 0xF8627820, 1) \ + X("LDR X?, #?", "LDR X0, #0x100", 0x58000800, 1) \ + X("LDR X?, #?", "LDR X0, [X1]", 0xF9400020, 0) \ + X("LDR W0, [X1, #0x18]", "LDR W0, [X1,#0x18]", 0xB9401820, 1) \ + X("LDRH W?, [X?,#?]", "LDRH W1, [X2,#0x10]", 0x79402041, 1) \ + X("LDRH W?, [X?,#?]", "STRH W1, [X2,#0x10]", 0x79002041, 0) \ + X("LDRH W?, [X?,#?]", "LDRB W1, [X2,#0x10]", 0x39404041, 0) \ + X("STR X?, [SP,#?]", "STR X5, [SP,#0x30]", 0xF9001BE5, 1) \ + X("STR X?, [SP,#?]", "STR X5, [X1,#0x30]", 0xF9001825, 0) \ + X("STR ?, [X?,#?]", "STR W5, [X1,#0x30]", 0xB9003025, 1) \ + X("STR W?, [X?,#0x1C]", "STR W5, [X1,#0x1C]", 0xB9001C25, 1) \ + X("STR X?, [X?,#?]!", "STR X0, [SP,#-0x10]!", 0xF81F0FE0, 1) \ + X("ADRP X?, #?", "ADRP X8, #0x5000", 0xB0000028, 1) \ + X("ADRP X?, #?", "ADR X8, #0x50", 0x10000288, 0) \ + X("ADRP X8, #?", "ADRP X9, #0x5000", 0xB0000029, 0) \ + X("ADR X?, #?", "ADR X1, #-0x40", 0x10FFFE01, 1) \ + X("ADD X?, X?, #?", "ADD X0, X1, #0x3A0", 0x910E8020, 1) \ + X("ADD X?, X?, #?", "SUB X0, X1, #0x3A0", 0xD10E8020, 0) \ + X("ADD X?, X?, #?", "ADD X0, X1, X2", 0x8B020020, 0) \ + X("ADD X?, X?, #?", "ADDS X0, X1, #1", 0xB1000420, 0) \ + X("ADD W?, W?, #?", "ADD W0, W1, #1", 0x11000420, 1) \ + X("ADD X?, X?, #?, LSL#12", "ADD X0, X1, #1, LSL#12", 0x91400420, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, X2", 0x8B020020, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, X3", 0x8B030020, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, #2", 0x91000820, 0) \ + X("SUB SP, SP, #?", "SUB SP, SP, #0x80", 0xD10203FF, 1) \ + X("SUB X?, X?, #?", "ADD X0, X1, #1", 0x91000420, 0) \ + X("SUBS W?, W?, #?", "SUBS W0, W1, #1", 0x71000420, 1) \ + X("SUBS W?, W?, #?", "SUB W0, W1, #1", 0x51000420, 0) \ + X("ADD X0, X1, #1, LSL #12", "ADD X0, X1, #1, LSL#12", 0x91400420, 1) \ + X("ADD X0, X1, #1, LSL #12", "ADD X0, X1, #1", 0x91000420, 0) \ + X("BL ?", "BL #0x2000", 0x94000800, 1) \ + X("BL #?", "BL #0x2000", 0x94000800, 1) \ + X("BL #?", "B #0x2000", 0x14000800, 0) \ + X("B #?", "B #-0x10", 0x17FFFFFC, 1) \ + X("B ?", "BL #0x10", 0x94000004, 0) \ + X("CBZ ?, #?", "CBZ W0, #0x20", 0x34000100, 1) \ + X("CBZ ?, #?", "CBNZ W0, #0x20", 0x35000100, 0) \ + X("CBNZ X?, #?", "CBNZ X1, #-0x20", 0xB5FFFF01, 1) \ + X("CBZ X?, #?", "CBZ W1, #0x20", 0x34000101, 0) \ + X("MOVZ W?, #?", "MOVZ W0, #0x10", 0x52800200, 1) \ + X("MOVZ W?, #?", "MOVK W0, #0x10", 0x72800200, 0) \ + X("MOVK X?, #?, LSL#?", "MOVK X0, #0x10, LSL#16", 0xF2A00200, 1) \ + X("MOVZ X?, #0x10", "MOVZ X0, #0x10, LSL#16", 0xD2A00200, 0) \ + X("ORR X?, X?, X?", "ORR X0, X1, X2", 0xAA020020, 1) \ + X("ORR X?, X?, #?", "ORR X0, X1, #0xFF", 0xB2401C20, 1) \ + X("ORR X?, X?, X?", "AND X0, X1, X2", 0x8A020020, 0) \ + X("ORR X0, XZR, X1", "MOV X0, X1", 0xAA0103E0, 1) \ + X("LDP X?, X?, [SP,#?]", "LDP X29, X30, [SP,#0x10]", 0xA9417BFD, 1) \ + X("LDP X?, X?, [SP,#?]", "STP X29, X30, [SP,#0x10]", 0xA9017BFD, 0) \ + X("LDP X?, X?, [SP],#?", "LDP X29, X30, [SP],#0x30", 0xA8C37BFD, 1) \ + X("LDP X?, X?, [SP,#?]!", "LDP X29, X30, [SP,#-0x30]!", 0xA9FD7BFD, 1) \ + X("BR X?", "BR X16", 0xD61F0200, 1) \ + X("BR X?", "BLR X16", 0xD63F0200, 0) \ + X("BLR X?", "BLR X8", 0xD63F0100, 1) \ + X("RET", "RET X1", 0xD65F0020, 0) \ + X("NOP", "NOP", 0xD503201F, 1) \ + X("RET", "RETAA", 0xD65F0BFF, 0) \ + X("LDR X?, [X29,#-8]", "LDUR X3, [X29,#-8]", 0xF85F83A3, 1) \ + X("LDR ?, [X29,#-8]", "LDUR W0, [X29,#-8]", 0xB85F83A0, 1) \ + X("LDR ?, [X29,#-8]", "LDR X0, [X29,#8]", 0xF94007A0, 0) \ + X("LDR X?, [X?,X?,LSL#?]", "LDR X2, [X3,X4]", 0xF8646862, 1) \ + X("LDUR X?, [X?,#?]", "LDUR X0, [X1,#0x10]", 0xF8410020, 1) \ + X("LDUR X?, [X?,#?]", "LDR X0, [X1,#0x10]", 0xF9400820, 0) \ + X("LDUR X?, [X?,#?]", "LDR X0, [X1,#8]!", 0xF8408C20, 0) \ + X("STUR ?, [X?,#?]", "STUR W0, [X29,#-4]", 0xB81FC3A0, 1) \ + X("STUR ?, [X?,#?]", "LDUR W0, [X29,#-4]", 0xB85FC3A0, 0) \ + X("LDRB W?, [X?,#?]", "LDRB W0, [X1,#0x10]", 0x39404020, 1) \ + X("LDRB W?, [X?,#?]", "LDRH W0, [X1,#0x10]", 0x79402020, 0) \ + X("LDRB W?, [X?,#?]", "STRB W0, [X1,#0x10]", 0x39004020, 0) \ + X("LDRB W?, [X?,X?,LSL#?]", "LDRB W0, [X1,X2,LSL#0]", 0x38627820, 1) \ + X("LDRB W?, [X?,X?,LSL#?]", "LDRB W0, [X1,X2]", 0x38626820, 1) \ + X("STRB ?, [X?,#?]", "STRB WZR, [X1]", 0x3900003F, 1) \ + X("STRH W?, [X?,#?]!", "STRH W0, [X1,#-2]!", 0x781FEC20, 1) \ + X("STRH W?, [X?,#?]!", "STRH W0, [X1,#0x10]", 0x79002020, 0) \ + X("LDRSW X?, [X?,#?]", "LDRSW X0, [X1,#0x10]", 0xB9801020, 1) \ + X("LDRSW X?, [X?,#?]", "LDR W0, [X1,#0x10]", 0xB9401020, 0) \ + X("CMP X?, #?", "CMP X0, #0x10", 0xF100401F, 1) \ + X("CMP X?, #?", "SUBS X1, X0, #0x10", 0xF1004001, 0) \ + X("CMP X?, #?", "CMP W0, #1", 0x7100041F, 0) \ + X("CMP ?, #?", "CMP W0, #1", 0x7100041F, 1) \ + X("CMP W?, W?", "CMP W0, W1", 0x6B01001F, 1) \ + X("CMP W?, W?", "CMN W0, W1", 0x2B01001F, 0) \ + X("CMP X?, X?, LSL#?", "CMP X0, X1, LSL#2", 0xEB01081F, 1) \ + X("CMN W?, #?", "CMN W0, #1", 0x3100041F, 1) \ + X("TST W?, #?", "TST W0, #0xFF", 0x72001C1F, 1) \ + X("TST W?, #?", "ANDS W1, W0, #0xFF", 0x72001C01, 0) \ + X("TST X?, X?", "TST X0, X1", 0xEA01001F, 1) \ + X("AND X?, X?, #?", "AND X0, X1, #0xFF", 0x92401C20, 1) \ + X("AND X?, X?, #?", "ORR X0, X1, #0xFF", 0xB2401C20, 0) \ + X("EOR X?, X?, X?", "EOR X0, X1, X2", 0xCA020020, 1) \ + X("EOR X?, X?, X?", "ORR X0, X1, X2", 0xAA020020, 0) \ + X("ANDS X?, X?, X?, LSR#?", "ANDS X0, X1, X2, LSR#3", 0xEA420C20, 1) \ + X("B.EQ #?", "B.EQ #0x10", 0x54000080, 1) \ + X("B.EQ #?", "B.NE #0x10", 0x54000081, 0) \ + X("B.? #?", "B.GT #0x20", 0x5400010C, 1) \ + X("B.? #?", "B #0x10", 0x14000004, 0) \ + X("B.? ?", "CBZ W0, #0x10", 0x34000080, 0) \ + X("B.HS ?", "B.CS #0x100", 0x54000802, 1) \ + X("EOR X0, X1, X2, LSR #?", "EOR X0, X1, X2, LSR #3", 0xCA420C20, 1) \ + X("SUB SP, SP, #?", "SUB SP, SP, #1, LSL #12", 0xD14007FF, 1) \ + X("SUB SP, SP, #?", "SUB SP, SP, #0x20", 0xD10083FF, 1) \ + X("CMP X0, #?", "CMP X0, #1, LSL #12", 0xF140041F, 1) \ + X("MOV X0, #?", "MOVN X0, #5", 0x928000A0, 1) \ + X("MOV X0, #?", "MOVK X0, #5", 0xF28000A0, 0) \ + X("LDP X29, X30, [SP]", "LDP X29, X30, [SP],#0x20", 0xA8C27BFD, 0) \ + X("ADD ?, X1, #8", "ADD W0, W1, #8", 0x11002020, 0) \ + X("STP ?, X30, [SP,#-0x10]!", "STP W29, W30, [SP,#-8]!", 0x29BF7BFD, 0) + +struct Word { uint32_t pat, mask; }; +template static Word word(const P& p, size_t idx = 0) { + Word w; + memcpy(&w.pat, p.pattern() + idx * 4, 4); + memcpy(&w.mask, p.mask() + idx * 4, 4); + return w; +} + +static void concrete() { + int n = 0; +#define CASE(text, enc) { \ + const auto w = word(ARM_PATTERN(text)); \ + if (w.pat != (enc) || w.mask != 0xFFFFFFFF) { \ + printf("FAIL %s: pat=%08X mask=%08X, want %08X/FFFFFFFF\n", text, w.pat, w.mask, (enc)); exit(1); \ + } ++n; } + CONCRETE(CASE) +#undef CASE + printf(" concrete encodings ok (%d)\n", n); +} + +static void wildcards() { + int n = 0; +#define CASE(ptext, itext, enc, want) { \ + const auto w = word(ARM_PATTERN(ptext)); \ + if ((((enc) & w.mask) == w.pat) != (want)) { \ + printf("FAIL %s vs %s: should%s match (pat=%08X mask=%08X enc=%08X)\n", ptext, itext, (want) ? "" : " not", w.pat, w.mask, (enc)); exit(1); \ + } ++n; } + WILDCARD(CASE) +#undef CASE + printf(" wildcard matching ok (%d)\n", n); +} + +// Forms the encoder doesn't support become a whole-word wildcard (mask 0), which the IDA plugin warns about, +// rather than a pattern that can never match +static void unsupported() { + CHECK(word(ARM_PATTERN("LDR Q0, [X1,#0x10]")).mask == 0); + CHECK(word(ARM_PATTERN("STR D0, [SP,#8]")).mask == 0); + CHECK(word(ARM_PATTERN("STP D9, D8, [SP,#-0x20]!")).mask == 0); + CHECK(word(ARM_PATTERN("LDR W0, [X1,#0x4000]")).mask == 0); // past the unsigned offset's range + CHECK(word(ARM_PATTERN("LDUR X0, [X1,#0x100]")).mask == 0); // past imm9 + CHECK(word(ARM_PATTERN("B.XX #0x10")).mask == 0); + puts(" unsupported forms ok"); +} + +template static bool same(const A& a, const B& b) { + return a.length() == b.length() && a.offset() == b.offset() && a.deref() == b.deref() && a.aligned() == b.aligned() + && !memcmp(a.pattern(), b.pattern(), a.length()) && !memcmp(a.mask(), b.mask(), a.length()); +} + +// Each line is parsed on its own: no instruction may turn into a wildcard because of what follows it +static void line_formats() { + constexpr auto canon = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + CHECK(canon.length() == 12 && canon.offset() == 4 && canon.deref() && canon.aligned()); + CHECK(word(canon, 0).pat == 0xD503201F && word(canon, 1).pat == 0xB9401908 && word(canon, 2).pat == 0xD65F03C0); + for (size_t k = 0; k < 3; ++k) CHECK(word(canon, k).mask == 0xFFFFFFFF); + CHECK(same(canon, ARM_PATTERN("\tNOP\r\n\tX LDR W8, [X8,#0x18]\r\n\tRET\r\n/d"))); // tabs, CRLF + CHECK(same(canon, ARM_PATTERN(" NOP \n\n X LDR W8, [X8, #0x18] \n\n RET\n/d"))); // padding, blank lines + CHECK(same(canon, ARM_PATTERN("nop\nx ldr w8, [x8, #0x18]\nret\n/d"))); // lowercase, as lldb prints it + CHECK(same(ARM_PATTERN("ADD X0, X1, #1, LSL#12"), ARM_PATTERN("ADD X0, X1, #1, LSL #12"))); + CHECK(same(ARM_PATTERN("BL #?"), ARM_PATTERN("BL ?"))); + CHECK(same(ARM_PATTERN("NOP\nRET"), ARM_PATTERN("NOP\nRET\n"))); + CHECK(ARM_PATTERN("NOP\nRET\n/a").length() == 8); + puts(" line formats ok"); +} + +// A function prologue as IDA prints it (STP, MOV): must skip a similar prologue, not treat STP/MOV as wildcards +static void prologue() { + alignas(4) static const uint32_t code[] = { + 0xD10083FF, 0xA9017BFD, 0x910043FD, 0xAA0103F3, 0xAA1E03E0, // SUB SP, SP, #0x20; STP; ADD X29; MOV; MOV + 0xA9BE4FF4, 0xA9017BFD, 0x910043FD, 0xAA0103F3, 0xAA1E03E0, // STP X20, X19, [SP,#-0x20]!; ... the one wanted + }; + const auto bytes = reinterpret_cast(code); +#define PROLOGUE "STP X20, X19, [SP,#?]!\n" \ + "STP X29, X30, [SP,#?]\n" \ + "ADD X29, SP, #0x10\n" \ + "MOV X19, X1\n" \ + "MOV X0, X30" + CHECK(ARM_PATTERN(PROLOGUE).find(bytes, sizeof code) == code + 5); + CHECK(patterns::RuntimeArmPattern(PROLOGUE).find(bytes, sizeof code) == code + 5); +#undef PROLOGUE + puts(" prologue ok"); +} + +// RuntimeArmPattern: the same parse as ArmPattern, from text only known at runtime +static void runtime_patterns() { + int n = 0; +#define CASE(text, ...) { CHECK(same(ARM_PATTERN(text), patterns::RuntimeArmPattern(text))); ++n; } + CONCRETE(CASE) +#undef CASE +#define CASE(ptext, ...) { CHECK(same(ARM_PATTERN(ptext), patterns::RuntimeArmPattern(ptext))); ++n; } + WILDCARD(CASE) +#undef CASE + constexpr auto canon = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + CHECK(same(canon, patterns::RuntimeArmPattern("nop\r\n x ldr w8, [x8, #0x18]\n\nret\n/d"))); + CHECK(same(ARM_PATTERN("NOP\nRET\n/a"), patterns::RuntimeArmPattern("NOP\nRET\n/a"))); + const char unterminated[] = { 'N', 'O', 'P', '\n', 'R', 'E', 'T', 'X' }; // the length bounds the text + CHECK(same(ARM_PATTERN("NOP\nRET"), patterns::RuntimeArmPattern(unterminated, 7))); + CHECK(patterns::RuntimeArmPattern("").length() == 0 && patterns::RuntimeArmPattern("/d").length() == 0); + printf(" runtime patterns ok (%d)\n", n); +} + +// ArmXORPattern: the same encoding and results as ArmPattern, different stored words per definition site +static void xor_patterns() { + constexpr auto plain = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + constexpr auto a = "NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"_armxorpattern; + constexpr auto b = "NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"_armxorpattern; + auto m = ARM_XOR_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); // deliberately not constexpr: still built at compile time + static_assert(a.length() == 12 && a.offset() == 4 && a.deref() && a.aligned()); + CHECK(memcmp(a.pattern(), b.pattern(), 12) != 0 && memcmp(a.pattern(), m.pattern(), 12) != 0); + CHECK(memcmp(a.mask(), b.mask(), 12) != 0 && memcmp(a.mask(), plain.mask(), 12) != 0); + for (size_t k = 0; k < 12; ++k) CHECK(a[k] == plain[k] && m[k] == plain[k] && a.unmask(k) == plain.mask()[k]); + + alignas(4) static const uint32_t code[] = { 0x52800000, 0xD503201F, 0xB9401908, 0xD65F03C0, 0 }; + const auto bytes = reinterpret_cast(code); + CHECK(a.match(bytes, sizeof code) == bytes + 4); + CHECK(b.match(bytes, sizeof code) == bytes + 4); + CHECK(m.match(bytes, sizeof code) == plain.match(bytes, sizeof code)); + CHECK(ARM_XOR_PATTERN("NOP\nX LDR W?, [X?,#?]").find(bytes, sizeof code) == bytes + 8); + puts(" xor patterns ok"); +} + +// /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size +static void deref_synthetic() { + static const uint32_t far_code[] = { 0x528EEEE9, 0x90480000, 0x91004000, 0 }; // MOVZ W9, #0x7777; ADRP X0, #0x90000000; ADD X0, X0, #0x10 + const auto far_page = reinterpret_cast(&far_code[1]) & ~uintptr_t(0xFFF); + constexpr auto adrl = ARM_PATTERN("MOVZ W9, #0x7777\nX ADRP X?, #?\n/d"); + CHECK(adrl.find(reinterpret_cast(far_code), sizeof far_code) == far_page + 0x90000000 + 0x10); + + static const uint32_t neg_code[] = { 0x528EEEE9, 0x90B80000, 0 }; // ADRP X0, #-0x90000000 + const auto neg_page = reinterpret_cast(&neg_code[1]) & ~uintptr_t(0xFFF); + CHECK(adrl.find(reinterpret_cast(neg_code), sizeof neg_code) == neg_page - 0x90000000); + + static const uint32_t adr_back[] = { 0x528EEEE9, 0x10FFFFE3, 0 }; // ADR X3, #-0x4 + CHECK(ARM_PATTERN("MOVZ W9, #0x7777\nX ADR X?, #?\n/d").find(reinterpret_cast(adr_back), sizeof adr_back) == adr_back); + + // ADRP X0, #0x1000; ADD X1, X0, #0x10; LDR X2, [X1,#8]: ADD into another register, then a load through it + static const uint32_t chain[] = { 0x528EEEE9, 0xB0000000, 0x91004001, 0xF9400422, 0 }; + const auto page = reinterpret_cast(&chain[1]) & ~uintptr_t(0xFFF); + const auto bytes = reinterpret_cast(chain); + CHECK(adrl.find(bytes, sizeof chain) == page + 0x1000 + 0x10 + 8); + CHECK(adrl.find(bytes, 12) == page + 0x1000 + 0x10); // the LDR is past the end: not read + CHECK(adrl.find(bytes, 8) == page + 0x1000); // ADRP is the last word scanned + const patterns::RuntimeArmPattern rt("MOVZ W9, #0x7777\nX ADRP X?, #?\n/d"); + bool is_value = true; + CHECK(rt.resolve(rt.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(page + 0x1000 + 0x10 + 8)); + CHECK(!is_value); + + // Values rather than addresses: the immediate of MOVZ, the offset of a lone LDR + const auto movz = ARM_PATTERN("X MOVZ W9, #0x7777\n/d"); + CHECK(movz.resolve(movz.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(0x7777) && is_value); + const auto ldr = ARM_PATTERN("ADD X1, X0, #0x10\nX LDR X2, [X1,#?]\n/d"); + CHECK(ldr.resolve(ldr.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(8) && is_value); + + // MOVZ/MOVN shifts, and /d values of loads/stores the old decoders missed (LDUR, LDRB, big STR offsets) + static const uint32_t vals[] = { + 0xD2A00020, // MOVZ X0, #1, LSL #16 + 0xD2C00020, // MOVZ X0, #1, LSL #32 + 0x92800000, // MOVN X0, #0 (MOV X0, #-1) + 0xF85F83A0, // LDUR X0, [X29,#-8] + 0x39448C20, // LDRB W0, [X1,#0x123] + 0xF9200020, // STR X0, [X1,#0x4000] + }; + const int64_t want[] = { 0x10000, 0x100000000, -1, -8, 0x123, 0x4000 }; + for (size_t k = 0; k < 6; ++k) { + const patterns::RuntimeArmPattern p("X ?\n/d"); + const auto at = reinterpret_cast(&vals[k]); + CHECK(p.resolve(at, at + 4, &is_value) == reinterpret_cast(want[k]) && is_value); + } + // Branches and literal loads with imm19 resolve to their target + static const uint32_t imm19[] = { 0x54000100, 0xB4000100, 0x58000100 }; // B.EQ, CBZ X0, LDR X0 (literal): #0x20 + for (auto& w : imm19) { + const auto at = reinterpret_cast(&w); + CHECK(patterns::RuntimeArmPattern("X ?\n/d").resolve(at, at + 4) == at + 0x20); + } + + // ADRP X8, #0x5000 followed by: LDRB, LDRSW, pre-index with a negative offset, post-index, an unrelated + // ADRP between ADRP and its ADD, and NOPs in the chain + struct { uint32_t next[2]; int64_t add; } follow[] = { + { { 0x39448D00, 0 }, 0x123 }, // LDRB W0, [X8,#0x123] + { { 0xB9801100, 0 }, 0x10 }, // LDRSW X0, [X8,#0x10] + { { 0xF85F8D00, 0 }, -8 }, // LDR X0, [X8,#-8]! + { { 0xF8408500, 0 }, 0 }, // LDR X0, [X8],#8 + { { 0x90000009, 0x91004108 }, 0x10 }, // ADRP X9, #0; ADD X8, X8, #0x10 + { { 0xD503201F, 0x91004108 }, 0x10 }, // NOP; ADD X8, X8, #0x10 + { { 0xD503201F, 0x39448D00 }, 0x123 }, // NOP; LDRB W0, [X8,#0x123] + }; + for (auto& f : follow) { + alignas(4) uint32_t code[4] = { 0x528EEEE9, 0xB0000028, f.next[0], f.next[1] }; // MOVZ W9; ADRP X8, #0x5000 + const auto b = reinterpret_cast(code); + const auto pg = (reinterpret_cast(&code[1]) & ~uintptr_t(0xFFF)) + 0x5000; + CHECK(adrl.find(b, sizeof code) == pg + f.add); + } + puts(" /d on synthetic code ok"); +} + +#if defined(PATTERNS_ARM64) && defined(__APPLE__) +#define SYM(x) "_" x +#define ASM_LINE(text, ...) text "\n" +#define ASM_WILD(ptext, itext, ...) itext "\n" +__asm__(".data\n.p2align 2\n.arch_extension pauth\n.globl " SYM("asm_concrete") "\n" SYM("asm_concrete") ":\n" CONCRETE(ASM_LINE) + ".globl " SYM("asm_wildcard") "\n" SYM("asm_wildcard") ":\n" WILDCARD(ASM_WILD)); +extern "C" const uint32_t asm_concrete[], asm_wildcard[]; + +// The tables' encodings must still be what the assembler produces +static void assembler_agrees() { + size_t i = 0; +#define CASE(text, enc) if (asm_concrete[i++] != (enc)) { printf("FAIL table: %s assembles to %08X\n", text, asm_concrete[i - 1]); exit(1); } + CONCRETE(CASE) +#undef CASE + i = 0; +#define CASE(ptext, itext, enc, want) if (asm_wildcard[i++] != (enc)) { printf("FAIL table: %s assembles to %08X\n", itext, asm_wildcard[i - 1]); exit(1); } + WILDCARD(CASE) +#undef CASE + puts(" tables match the assembler ok"); +} + +// /d against code whose ADRP/ADD/LDR/BL operands were filled in by the linker +extern "C" int arm_target_data; +int arm_target_data = 0x1234; +extern "C" void arm_target_func() {} +extern "C" const uint32_t arm_reloc_code[]; +__asm__(".text\n.p2align 2\n.globl " SYM("arm_reloc_code") "\n" SYM("arm_reloc_code") ":\n" + " movz w9, #0x7777\n" + " adrp x0, " SYM("arm_target_data") "@PAGE\n" + " add x0, x0, " SYM("arm_target_data") "@PAGEOFF\n" + " movz w9, #0x6666\n" + " adrp x8, " SYM("arm_target_data") "@PAGE\n" + " add x1, x8, " SYM("arm_target_data") "@PAGEOFF\n" // different destination register + " movz w9, #0x5555\n" + " adrp x8, " SYM("arm_target_data") "@PAGE\n" + " ldr w2, [x8, " SYM("arm_target_data") "@PAGEOFF]\n" + " movz w9, #0x4444\n" + " bl " SYM("arm_target_func") "\n" + " movz w9, #0x3333\n" + " b " SYM("arm_target_func") "\n" + " ret\n"); + +static void deref_relocated() { + const auto code = reinterpret_cast(arm_reloc_code); + const size_t n = 14 * 4; + const void* data = &arm_target_data; + const void* func = reinterpret_cast(&arm_target_func); + CHECK(ARM_PATTERN("MOVZ W9, #0x7777\nX ADRP X?, #?\nADD X?, X?, #?\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x6666\nX ADRP X?, #?\nADD X?, X?, #?\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x5555\nX ADRP X?, #?\nLDR W?, [X?,#?]\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x4444\nX BL ?\n/d").find(code, n) == func); + CHECK(ARM_PATTERN("MOVZ W9, #0x3333\nX B #?\n/d").find(code, n) == func); + CHECK(ARM_PATTERN("MOVZ W9, #0x4444\nX BL ?").find(code, n) == code + 40); // no /d: the X address + puts(" /d on relocated code ok"); +} +#endif + +int main() { + concrete(); + wildcards(); + unsupported(); + line_formats(); + prologue(); + runtime_patterns(); + xor_patterns(); + deref_synthetic(); +#if defined(PATTERNS_ARM64) && defined(__APPLE__) + assembler_agrees(); + deref_relocated(); +#endif + puts("all arm tests passed"); +} diff --git a/tests/test_artifacts.cpp b/tests/test_artifacts.cpp new file mode 100644 index 0000000..d3daaa2 --- /dev/null +++ b/tests/test_artifacts.cpp @@ -0,0 +1,152 @@ +// Compile-time patterns must leave nothing behind in the binary: no pattern text for any of them, and no plaintext +// bytes, words or masks for the XOR ones. tests/CMakeLists.txt builds this at every optimization level and with LTO, +// with debug info, as C++17 (macros) and C++20 (macros and literals). Exit code 0 = pass. +// Needles are written reversed and turned around at runtime, so this file doesn't put them in the binary itself. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +static std::vector bin; + +static bool contains(const void* needle, size_t len) { + const auto n = static_cast(needle); + return std::search(bin.begin(), bin.end(), n, n + len) != bin.end(); +} + +// Reversed text, turned around at runtime. Read through volatile: otherwise the optimizer reverses the literal at +// compile time and puts the needle in the binary itself (seen with clang -O2, C++20) +static std::string forward(const char* reversed) { + const volatile char* v = reversed; + std::string s; + for (size_t i = 0; v[i]; ++i) s.insert(s.begin(), v[i]); + return s; +} + +// Space-separated hex bytes, ? as 0 (how wildcards are stored) +static std::vector hex_bytes(const std::string& text) { + std::vector out; + std::istringstream in(text); + for (std::string tok; in >> tok;) + out.push_back(tok == "?" ? 0 : static_cast(std::stoul(tok, nullptr, 16))); + return out; +} + +// Needle text, and code containing the pattern for it to find +struct Case { + std::string text; + // Fixed size, like most real scans of a known buffer: with a size the optimizer can see, it unrolls and folds more + uint8_t code[64] = {}; + std::vector bytes, masks; // masks: arm64 only + size_t at = 0; +}; + +static Case byte_case(const char* reversed) { + Case c; + c.text = forward(reversed); + c.bytes = hex_bytes(c.text); + memset(c.code, 0x90, sizeof c.code); + c.at = 3; + memcpy(c.code + c.at, c.bytes.data(), c.bytes.size()); + return c; +} + +static Case arm_case(const char* reversed) { + Case c; + c.text = forward(reversed); + const patterns::RuntimeArmPattern rt(c.text.c_str()); + c.bytes.assign(rt.pattern(), rt.pattern() + rt.length()); + c.masks.assign(rt.mask(), rt.mask() + rt.length()); + for (size_t i = 0; i < sizeof c.code; i += 4) memcpy(&c.code[i], "\x1F\x20\x03\xD5", 4); // NOPs around it + c.at = 8; + memcpy(c.code + c.at, c.bytes.data(), c.bytes.size()); + return c; +} + +// No text in the binary (each line on its own for arm64, so partly folded text is caught too); for XOR patterns no +// plaintext bytes or masks either (masks only as a whole: single mask words like FFFFFFFF are everywhere). The +// pattern must still have found its bytes +static bool check(const char* name, const Case& c, const uint8_t* found, bool xored) { + if (found != c.code + c.at) { printf("FAIL %s: didn't find its bytes\n", name); return false; } + for (size_t a = 0, b; a < c.text.size(); a = b + 1) { + b = c.text.find('\n', a); + if (b == std::string::npos) b = c.text.size(); + if (contains(c.text.data() + a, b - a)) { printf("FAIL %s: pattern text in the binary\n", name); return false; } + } + // Folded plaintext may be split into 4-byte immediates (arm64 words always are), so each word on its own + for (size_t i = 0; xored && i + 4 <= c.bytes.size(); i += 4) + if (contains(&c.bytes[i], 4)) { printf("FAIL %s: plaintext bytes %zu..%zu in the binary\n", name, i, i + 3); return false; } + if (xored && !c.masks.empty() && contains(c.masks.data(), c.masks.size())) { printf("FAIL %s: plaintext masks in the binary\n", name); return false; } + return true; +} + +// Each pattern is unique, and its needle is its text reversed. The XOR arm64 ones use mask shapes no plain +// pattern here has: plain patterns' masks are in the binary by design. The XOR byte patterns have no wildcards, +// so each of their 4-byte words is distinctive +#define MACRO_CASES(X) \ + X("COMPILETIME_PATTERN", byte_case, COMPILETIME_PATTERN("1A 2B ? 4D 5E 6F 71 82"), "28 17 F6 E5 D4 ? B2 A1", false) \ + X("XOR_PATTERN", byte_case, XOR_PATTERN("A1 B2 C3 D4 E5 F6 07 18"), "81 70 6F 5E 4D 3C 2B 1A", true) \ + X("ARM_PATTERN", arm_case, ARM_PATTERN("MOVZ W4, #0x1357\nMOVK W4, #0x2468, LSL #16\nLDR W?, [X4,#?]"), \ + "]?#,4X[ ,?W RDL\n61# LSL ,8642x0# ,4W KVOM\n7531x0# ,4W ZVOM", false) \ + X("ARM_XOR_PATTERN", arm_case, ARM_XOR_PATTERN("MOVZ W?, #0x7531\nMOVK W5, #0x8642, LSL #16\nSTR W?, [X5,#0x2C]"), \ + "]C2x0#,5X[ ,?W RTS\n61# LSL ,2468x0# ,5W KVOM\n1357x0# ,?W ZVOM", true) +#ifdef PATTERNS_LITERALS +#define LITERAL_CASES(X) \ + X("_ctpattern", byte_case, "3C 4D ? 6F 71 82 93 A4"_ctpattern, "4A 39 28 17 F6 ? D4 C3", false) \ + X("_xorpattern", byte_case, "C3 D4 E5 F6 17 28 39 4A"_xorpattern, "A4 93 82 71 6F 5E 4D 3C", true) \ + X("_armpattern", arm_case, "MOVZ W6, #0x1A2B\nMOVK W6, #0x3C4D, LSL #16\nLDR W?, [X6,#?]"_armpattern, \ + "]?#,6X[ ,?W RDL\n61# LSL ,D4C3x0# ,6W KVOM\nB2A1x0# ,6W ZVOM", false) \ + X("_armxorpattern", arm_case, "MOVZ W7, #0x2B1A\nMOVK W?, #0x4D3C, LSL #16\nSTR W?, [X7,#0x34]"_armxorpattern, \ + "]43x0#,7X[ ,?W RTS\n61# LSL ,C3D4x0# ,?W KVOM\nA1B2x0# ,7W ZVOM", true) +#else +#define LITERAL_CASES(X) +#endif + +#ifdef _MSC_VER +#define NOINLINE __declspec(noinline) +#else +#define NOINLINE __attribute__((noinline)) +#endif + +// All scans in one function with nothing else in it, as a project's signature file does it: the shape in which +// optimizers inline find() and fold constants, e.g. an XOR pattern's decode back into plaintext. Locals, not +// constexpr: the macros and literals must build at compile time on their own. Not static: gcc treats a static +// function only reachable from main() as run once and stops inlining into it, which would hide such a leak +NOINLINE void scan_all(const uint8_t* const* code, size_t size, const uint8_t** found) { + size_t i = 0; +#define SCAN(name, make_case, def, reversed, xored) { auto p = def; found[i] = p.find(code[i], size); ++i; } + MACRO_CASES(SCAN) + LITERAL_CASES(SCAN) +#undef SCAN +} + +int main(int, char** argv) { + FILE* f = fopen(argv[0], "rb"); + if (!f) { printf("FAIL: can't open %s\n", argv[0]); return 1; } + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); + fclose(f); + + struct Entry { const char* name; Case c; bool xored; }; + std::vector cases; +#define ADD(name, make_case, def, reversed, xored) cases.push_back({ name, make_case(reversed), xored }); + MACRO_CASES(ADD) + LITERAL_CASES(ADD) +#undef ADD + std::vector code, found(cases.size()); + for (const auto& e : cases) code.push_back(e.c.code); + scan_all(code.data(), sizeof cases[0].c.code, found.data()); + // Every leak reported, not just the first + bool ok = true; + for (size_t i = 0; i < cases.size(); ++i) ok &= check(cases[i].name, cases[i].c, found[i], cases[i].xored); + if (!ok) return 1; + printf("%zu patterns: no pattern text, no XOR plaintext\n", cases.size()); +} diff --git a/tests/test_cpp17.cpp b/tests/test_cpp17.cpp new file mode 100644 index 0000000..ec073dd --- /dev/null +++ b/tests/test_cpp17.cpp @@ -0,0 +1,30 @@ +// The pre-C++20 path: macros instead of literals. Built as C++17 by tests/CMakeLists.txt. +#include +#include +#include +#include +#include +#include + +int main() { + static const uint8_t buf[] = { 1, 2, 0x48, 0x8B, 5, 6, 7, 8 }; + auto ct = COMPILETIME_PATTERN("48 8B ? ? 07"); + auto x1 = XOR_PATTERN("48 8B ? ? 07"); + auto x2 = XOR_PATTERN("48 8B ? ? 07"); + patterns::RuntimePattern rp("48 8B ? ? 07"); + alignas(4) static const uint32_t code[] = { 0xD503201F, 0xD65F03C0 }; // NOP; RET + const auto code_bytes = reinterpret_cast(code); + auto arm = ARM_PATTERN("NOP\nRET"); + auto arm_xor = ARM_XOR_PATTERN("NOP\nRET"); + patterns::RuntimeArmPattern arm_rt("NOP\nRET"); + const bool ok = ct.find(buf, 8) == buf + 2 + && x1.find(buf, 8) == buf + 2 + && x2.find(buf, 8) == buf + 2 + && rp.find(buf, 8) == buf + 2 + && arm.find(code_bytes, 8) == code_bytes + && arm_xor.find(code_bytes, 8) == code_bytes + && arm_rt.find(code_bytes, 8) == code_bytes + && memcmp(x1.pattern(), x2.pattern(), 5) != 0; // per-definition keys + puts(ok ? "c++17 macros ok" : "FAIL c++17 macros"); + return !ok; +} diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp new file mode 100644 index 0000000..4837b67 --- /dev/null +++ b/tests/test_patterns.cpp @@ -0,0 +1,226 @@ +// Correctness tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. +#include +#include +#include +#include +#include "multi_scan.hpp" +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +// Reference implementation: every position, every byte +static const uint8_t* naive(const uint8_t* b, size_t n, const uint8_t* p, const uint8_t* m, size_t len, size_t step) { + for (size_t i = 0; i + len <= n; ++i) { + if (reinterpret_cast(b + i) % step) continue; + size_t j = 0; + while (j < len && !((b[i + j] ^ p[j]) & m[j])) ++j; + if (j == len) return b + i; + } + return nullptr; +} + +// Arbitrary bit masks (like ArmPattern produces), usable on any platform +struct MaskPattern : patterns::Pattern { + std::vector p, m; + MaskPattern(std::vector p_, std::vector m_, bool align) : p(std::move(p_)), m(std::move(m_)) { + length_ = static_cast(p.size()); + align_ = align; + } + const uint8_t* pattern() const override { return p.data(); } + const uint8_t* mask() const override { return m.data(); } +}; + +static void edge_cases() { + uint8_t t[16] = {}; + memcpy(t + 8, "\xAA\xBB\xCC\xDD\xEE\xFF\x11\x22", 8); + CHECK("AA BB CC DD EE FF 11 22"_rtpattern.find(t, 16) == t + 8); // match in the last position + CHECK("EE FF 11 22"_rtpattern.find(t, 16) == t + 12); // short pattern at the very end + CHECK("AA BB CC DD EE FF 11 22"_rtpattern.find(t, 4) == nullptr); // buffer smaller than pattern + CHECK("? ? ? ?"_rtpattern.find(t, 16) == t); // all wildcards + CHECK("EE FF 11 22"_rtpattern.length() == 4); // no padding + bool threw = false; + try { "48 8G"_rtpattern; } catch (const std::logic_error&) { threw = true; } + CHECK(threw); // invalid hex rejected + { const char s[] = "48 8B X4"; patterns::RuntimePattern rp(s, 8); CHECK(rp.length() == 2); } // X at the end, no overrun + // Wildcards: ? and ?? are a whole byte, ? beside a hex digit a nibble + { + const patterns::RuntimePattern w("? ?? 4? ?8 4F"); + const uint8_t pat[] = { 0, 0, 0x40, 0x08, 0x4F }, msk[] = { 0, 0, 0xF0, 0x0F, 0xFF }; + CHECK(w.length() == 5 && !memcmp(w.pattern(), pat, 5) && !memcmp(w.mask(), msk, 5)); + constexpr auto ct = COMPILETIME_PATTERN("? ?? 4? ?8 4F"); + CHECK(ct.length() == 5 && !memcmp(ct.pattern(), pat, 5) && !memcmp(ct.mask(), msk, 5)); + const auto x = XOR_PATTERN("? ?? 4? ?8 4F"); + for (size_t i = 0; i < 5; ++i) CHECK(x[i] == pat[i] && x.unmask(i) == msk[i]); + const uint8_t hit[] = { 0x11, 0x22, 0x4A, 0xB8, 0x4F }, miss[] = { 0x11, 0x22, 0x5A, 0xB8, 0x4F }; + CHECK(w.find(hit, 5) == hit && x.find(hit, 5) == hit); + CHECK(!w.find(miss, 5) && !x.find(miss, 5)); + const char s[] = "48 ?"; patterns::RuntimePattern rp(s, 4); // trailing ?, not terminated + CHECK(rp.length() == 2 && rp.mask()[1] == 0); + threw = false; + try { "4G ?"_rtpattern; } catch (const std::logic_error&) { threw = true; } + CHECK(threw); + } + + // X offset, /r (relative value) and /d (rip-relative deref) on a call: E8 + alignas(8) uint8_t code[16] = { 0x90, 0xE8, 0x10, 0x00, 0x00, 0x00, 0xC3 }; + CHECK("E8 X ? ? ? ? C3"_rtpattern.find(code, 16) == code + 2); + // x86 semantics on any host, as a disassembler plugin scanning x86 code needs (arm64 has no /r) + for (auto a : { patterns::arch::x86, patterns::arch::x64 }) { + const patterns::RuntimePattern rel("E8 X4 ? ? ? ? C3 /r", a), deref("E8 X4 ? ? ? ? C3 /d", a); + CHECK(rel.find(code, 16) == 0x10); + CHECK(deref.find(code, 16) == code + 6 + 0x10); + bool is_value = false; + CHECK(rel.resolve(rel.match(code, 16), code + 16, &is_value) == reinterpret_cast(0x10) && is_value); + CHECK(deref.resolve(deref.match(code, 16), code + 16, &is_value) == code + 6 + 0x10 && !is_value); + CHECK(rel.target() == a); + } + // arm64 semantics for byte patterns: /d decodes the instruction at X (BL #0x1000), X isn't parsed + alignas(4) const uint32_t bl[] = { 0xD503201F, 0x94000400 }; + const auto bl_bytes = reinterpret_cast(bl); + CHECK(patterns::RuntimePattern("1F 20 03 D5 X ? ? ? 94 /d", patterns::arch::arm64).find(bl_bytes, 8) == bl_bytes + 4 + 0x1000); + // /a steps by the target's alignment: 8 for x64, 4 for x86 and arm64 + alignas(8) uint8_t al[24] = {}; + al[12] = 0xAB; + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::x64).find(al, 24) == nullptr); + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::x86).find(al, 24) == al + 12); + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::arm64).find(al, 24) == al + 12); + threw = false; + try { patterns::RuntimePattern("E8 X4 ? ? ? ? /r8", patterns::arch::x86); } catch (const std::logic_error&) { threw = true; } + CHECK(!threw); // 8 isn't a size on x86, so it's ignored like any other unknown flag + puts(" edge cases ok"); +} + +static void literals_and_macros() { + static const uint8_t buf[] = { 0, 1, 0xDE, 0xAD, 0xBE, 0xEF, 0x13, 0x37, 0xC0, 0xDE, 9 }; +#ifdef PATTERNS_LITERALS + // Same text, different definition sites -> different stored bytes, same results + constexpr auto a = "DE AD BE EF 13 37 C0 DE"_xorpattern; + constexpr auto b = "DE AD BE EF 13 37 C0 DE"_xorpattern; + constexpr auto c = "DE AD BE EF 13 37 C0 DE"_xorpattern; constexpr auto d = "DE AD BE EF 13 37 C0 DE"_xorpattern; + CHECK(memcmp(a.pattern(), b.pattern(), 8) != 0); + CHECK(memcmp(c.pattern(), d.pattern(), 8) != 0); + CHECK(memcmp(a.mask(), b.mask(), 8) != 0); // masks are XOR'd too + CHECK(a.find(buf, sizeof buf) == buf + 2); + CHECK(d.find(buf, sizeof buf) == buf + 2); + constexpr auto ct = "DE AD ? EF X 13"_ctpattern; + static_assert(ct.length() == 5 && ct.offset() == 4); + CHECK(ct.find(buf, sizeof buf) == buf + 6); + constexpr auto xw = "DE AD ? EF X 13"_xorpattern; + for (size_t k = 0; k < 5; ++k) CHECK(xw.unmask(k) == ct.mask()[k] && xw[k] == ct.pattern()[k]); + CHECK(xw.find(buf, sizeof buf) == buf + 6); +#endif + auto m1 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); // deliberately not constexpr: still built at compile time + auto m2 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); + CHECK(memcmp(m1.pattern(), m2.pattern(), 8) != 0); + CHECK(m2.find(buf, sizeof buf) == buf + 2); + CHECK(COMPILETIME_PATTERN("DE AD ? EF").find(buf, sizeof buf) == buf + 2); + puts(" literals and macros ok"); +} + +// The XOR pattern's plaintext and source text must not be in our own executable. The needle is stored +// scrambled and rebuilt at runtime, otherwise the test itself would put it in the binary. +// Random patterns with random bit masks against the naive reference: both scan modes, +// sizes around the SIMD block boundaries, odd base addresses +static void fuzz(unsigned rounds) { + std::mt19937 rng(7); + std::vector buf(300); + for (unsigned r = 0; r < rounds; ++r) { + for (auto& x : buf) x = rng() % 4 ? rng() % 3 : rng(); // small alphabet so partial matches are common + const size_t len = 1 + rng() % 24, n = rng() % 200, base = rng() % 9; + std::vector p(len), m(len); + for (size_t j = 0; j < len; ++j) { + const int kind = rng() % 4; + m[j] = kind == 0 ? 0 : kind == 1 ? static_cast(rng()) : 0xFF; + p[j] = (rng() % 2 ? rng() % 3 : rng()) & m[j]; + } + if (rng() % 3 == 0 && n >= len) { // plant a match + const size_t at = rng() % (n - len + 1); + for (size_t j = 0; j < len; ++j) + buf[base + at + j] = (buf[base + at + j] & ~m[j]) | p[j]; + } + const bool align = rng() % 4 == 0; + MaskPattern mp(p, m, align); + const auto got = mp.find(buf.data() + base, n); + const size_t align_step = mp.target() == patterns::arch::x64 ? 8 : 4; // see Pattern::align_size() + const auto want = naive(buf.data() + base, n, p.data(), m.data(), len, align ? align_step : 1); + if (got != want) { + printf("FAIL fuzz round %u: len=%zu n=%zu base=%zu align=%d\n", r, len, n, base, align); + exit(1); + } + } + printf(" fuzz ok (%u rounds)\n", rounds); +} + +// Every multi-pattern strategy must agree with plain find() calls, including /d results and matches +// that straddle chunk boundaries (small chunks force many boundaries) +static void multi_strategies() { + std::mt19937 rng(3); + std::vector mod(1 << 20); + for (auto& x : mod) x = static_cast(rng()); + std::vector owned; + std::vector pats; + for (int k = 0; k < 40; ++k) { + const size_t off = rng() % (mod.size() - 32); + std::string s; + char hex[4]; + for (int j = 0; j < 12; ++j) { + if (j == 3) s += "X "; + if (j >= 4 && j < 8) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", mod[off + j]); s += hex; } + } + s += k % 2 ? "/d" : ""; + // x86-style /d on random bytes; arm64 /d would try to decode them as instructions + owned.emplace_back(s.c_str(), patterns::arch::x64); + } + owned.emplace_back("01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F"); // absent + for (auto& p : owned) pats.push_back(&p); + std::vector want, got; + multi::sequential(mod.data(), mod.size(), pats, want); + multi::per_pattern(mod.data(), mod.size(), pats, got); + CHECK(got == want); + for (size_t chunk : { 7, 64, 4096, 1 << 20 }) { + got.assign(pats.size(), &got); // garbage, find_all must overwrite every entry + CHECK(patterns::find_all(mod.data(), mod.size(), pats.data(), pats.size(), got.data(), chunk) == pats.size() - 1); + CHECK(got == want); + multi::chunked(mod.data(), mod.size(), pats, got, 1, chunk); + CHECK(got == want); + multi::chunked(mod.data(), mod.size(), pats, got, 4, chunk); + CHECK(got == want); + } + puts(" multi-pattern strategies agree ok"); +} + +static void arm_patterns() { + constexpr auto adrl = "LDR W?, [X?,#0x18]\n" + "X ADRP X?, #?\n" + "ADD X?, X?, #?\n" + "STR W?, [SP,#?]\n" + "/d"_armpattern; + static_assert(adrl.length() == 16 && adrl.offset() == 4 && adrl.deref() && adrl.aligned()); + alignas(4) static const uint32_t code[] = { 0xD503201F, 0xD503201F, 0xD65F03C0, 0 }; // NOP NOP RET + constexpr auto ret = "NOP\nRET"_armpattern; + CHECK(ret.find(reinterpret_cast(code), sizeof code) == code + 1); + puts(" arm64 patterns ok"); +} + +int main() { +#ifdef PATTERNS_NO_SIMD + puts("scan path: scalar (PATTERNS_NO_SIMD)"); +#elif defined(PATTERNS_SSE2) + puts("scan path: SSE2"); +#elif defined(PATTERNS_NEON) + puts("scan path: NEON"); +#else + puts("scan path: scalar"); +#endif + edge_cases(); + literals_and_macros(); + fuzz(300000); + multi_strategies(); + arm_patterns(); + puts("all tests passed"); +} diff --git a/tests/test_tls.cpp b/tests/test_tls.cpp new file mode 100644 index 0000000..2381470 --- /dev/null +++ b/tests/test_tls.cpp @@ -0,0 +1,51 @@ +// patterns::find_all from a TLS callback: runs as the module is loaded, before the CRT is initialized and before any +// static constructor or main. Scans this executable's own image. Windows only. +#include +#include +#ifdef _MSC_VER +#include + +// Data to find in our own image. The patterns are XOR'd, so their stored bytes can't match themselves +extern "C" const uint8_t tls_target[] = { 0x5E, 0xC7, 0x13, 0x9A, 0x40, 0xD2, 0x7B, 0xE1, 0x36, 0xAF, 0x08, 0x95 }; + +inline constexpr auto whole = "5E C7 13 9A 40 D2 7B E1 36 AF 08 95"_xorpattern; +inline constexpr auto with_offset = "5E C7 ? 9A X 40 D2 7B E1"_xorpattern; +inline constexpr auto absent = "5E C7 13 9A 40 D2 7B E1 36 AF 08 96"_xorpattern; +inline constexpr const patterns::Pattern* tls_patterns[] = { &whole, &with_offset, &absent }; + +static void* tls_results[3]; +static size_t tls_found; +static bool callback_ran, callback_saw_crt; + +// Dynamic initializer (a call the compiler can't fold into static data): nonzero once the CRT is up +static const DWORD crt_initialized = GetCurrentProcessId(); + +static void NTAPI on_tls(PVOID module, DWORD reason, PVOID) { + if (reason != DLL_PROCESS_ATTACH) return; + callback_ran = true; + callback_saw_crt = crt_initialized != 0; + const auto base = static_cast(module); + const auto nt = reinterpret_cast(base + reinterpret_cast(base)->e_lfanew); + tls_found = patterns::find_all(base, nt->OptionalHeader.SizeOfImage, tls_patterns, 3, tls_results); +} + +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#pragma comment(linker, "/INCLUDE:tls_callback") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#pragma comment(linker, "/INCLUDE:_tls_callback") +#endif +#pragma section(".CRT$XLB", read) +extern "C" __declspec(allocate(".CRT$XLB")) const PIMAGE_TLS_CALLBACK tls_callback = on_tls; + +int main() { + const bool ok = callback_ran && !callback_saw_crt && crt_initialized && tls_found == 2 + && tls_results[0] == tls_target && tls_results[1] == tls_target + 4 && !tls_results[2]; + printf("callback ran: %d, before CRT init: %d, found: %zu\n", callback_ran, !callback_saw_crt, tls_found); + puts(ok ? "tls ok" : "FAIL tls"); + return !ok; +} +#else +int main() { puts("TLS callbacks registered via MSVC sections; skipped"); } +#endif