From 827000dd44522633fae56a2414e92e1250f1d73a Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Tue, 1 Sep 2026 03:35:19 +0000 Subject: [PATCH] perf(debuginfo): sort loctab on packed (addr, index) keys canonicaliseLoctab() sorted the DWARF line table of every loaded object by handing an array of UInt indexes to VG_(ssort), whose comparison function dereferenced di->loctab[ix] for both operands: an indirect call plus two random accesses into a multi-megabyte table for every one of the O(n log n) comparisons. VG_(ssort) also exchanges elements whose size is not a multiple of the word size, such as these 4 byte indexes, one byte at a time. Sort a packed { Addr addr; UInt ix; } key array with a small specialised quicksort instead (median-of-3 pivot, insertion sort for short partitions, recursion into the smaller side so the stack depth stays O(log n)). The comparison is inlined, the key sits next to the index it moves, and swaps are whole-struct moves. The index array and the in-place permutation that follows are unchanged. Ties are broken by the original index, so the ordering of entries sharing an address is now deterministic instead of depending on quicksort's arbitrary placement of equal elements. --- coregrind/m_debuginfo/storage.c | 95 ++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/coregrind/m_debuginfo/storage.c b/coregrind/m_debuginfo/storage.c index b66339d9f..5f9dba470 100644 --- a/coregrind/m_debuginfo/storage.c +++ b/coregrind/m_debuginfo/storage.c @@ -2022,15 +2022,81 @@ static void canonicaliseSymtab ( struct _DebugInfo* di ) } -static DiLoc* sorting_loctab = NULL; -static Int compare_DiLoc_via_ix ( const void* va, const void* vb ) +/* (address, index) pair used to sort loctab. Sorting these directly, + rather than sorting an array of indexes into loctab, keeps the sort's + memory accesses local: the key it compares sits next to the index it + moves, instead of being a random access into a multi-megabyte loctab + for every one of the O(n log n) comparisons. */ +typedef + struct { Addr addr; UInt ix; } + DiLocSortKey; + +/* Strict order on DiLocSortKey. Keys are unique (the index breaks ties), + so this is a total order and equal elements never occur. */ +static inline Bool loc_key_lt ( const DiLocSortKey* a, const DiLocSortKey* b ) { - const DiLoc* a = &sorting_loctab[*(const UInt*)va]; - const DiLoc* b = &sorting_loctab[*(const UInt*)vb]; - if (a->addr < b->addr) return -1; - if (a->addr > b->addr) return 1; - return 0; + if (a->addr != b->addr) return a->addr < b->addr; + return a->ix < b->ix; } + +/* Quicksort specialised for DiLocSortKey: the comparison is inlined and + the elements are swapped as whole structs, unlike VG_(ssort), which + calls the comparison through a function pointer and (for elements + whose size is not a multiple of the word size, such as the 4 byte + indexes sorted here) exchanges them one byte at a time. + Small partitions are finished off with insertion sort, and the + recursion always descends into the smaller partition, so the stack + depth stays O(log n). */ +#define DILOC_SORT_INSERTION_MAX 12 +static void sort_loc_keys ( DiLocSortKey* arr, Word n ) +{ + Word i, j; + DiLocSortKey pivot, tmp; + + while (n > DILOC_SORT_INSERTION_MAX) { + /* Median of the first, middle and last element, ordering those + three on the way. Afterwards arr[0] < pivot < arr[n-1], so both + partitioning loops below have a sentinel and cannot run off the + ends of the array. */ + Word mid = n >> 1; + if (loc_key_lt(&arr[mid], &arr[0])) + { tmp = arr[mid]; arr[mid] = arr[0]; arr[0] = tmp; } + if (loc_key_lt(&arr[n-1], &arr[0])) + { tmp = arr[n-1]; arr[n-1] = arr[0]; arr[0] = tmp; } + if (loc_key_lt(&arr[n-1], &arr[mid])) + { tmp = arr[n-1]; arr[n-1] = arr[mid]; arr[mid] = tmp; } + /* Park the pivot in arr[1], out of the way of the partitioning. */ + tmp = arr[mid]; arr[mid] = arr[1]; arr[1] = tmp; + pivot = arr[1]; + + i = 1; j = n - 1; + for (;;) { + do { i++; } while (loc_key_lt(&arr[i], &pivot)); + do { j--; } while (loc_key_lt(&pivot, &arr[j])); + if (i > j) break; + tmp = arr[i]; arr[i] = arr[j]; arr[j] = tmp; + } + arr[1] = arr[j]; arr[j] = pivot; + + /* Recurse into the smaller side, iterate on the larger one. */ + if (j < n - i) { + sort_loc_keys(arr, j); + arr += i; n -= i; + } else { + sort_loc_keys(arr + i, n - i); + n = j; + } + } + + for (i = 1; i < n; i++) { + pivot = arr[i]; + for (j = i - 1; j >= 0 && loc_key_lt(&pivot, &arr[j]); j--) + arr[j+1] = arr[j]; + arr[j+1] = pivot; + } +} +#undef DILOC_SORT_INSERTION_MAX + static void sort_loctab_and_loctab_fndn_ix (struct _DebugInfo* di ) { /* We have to sort the array loctab by addr @@ -2040,13 +2106,18 @@ static void sort_loctab_and_loctab_fndn_ix (struct _DebugInfo* di ) arrays according to sort_ix. */ UInt *sort_ix = ML_(dinfo_zalloc)("di.storage.six", di->loctab_used*sizeof(UInt)); + DiLocSortKey *sort_keys = ML_(dinfo_zalloc)("di.storage.sk", + di->loctab_used + * sizeof(DiLocSortKey)); Word i, j, k; - for (i = 0; i < di->loctab_used; i++) sort_ix[i] = i; - sorting_loctab = di->loctab; - VG_(ssort)(sort_ix, di->loctab_used, - sizeof(*sort_ix), compare_DiLoc_via_ix); - sorting_loctab = NULL; + for (i = 0; i < di->loctab_used; i++) { + sort_keys[i].addr = di->loctab[i].addr; + sort_keys[i].ix = (UInt)i; + } + sort_loc_keys(sort_keys, di->loctab_used); + for (i = 0; i < di->loctab_used; i++) sort_ix[i] = sort_keys[i].ix; + ML_(dinfo_free)(sort_keys); // Permute in place, using the sort_ix. for (i=0; i < di->loctab_used; i++) {