From 7adb151c4fca6d3ba6df42772bb8522c43163d07 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 20:40:37 +0100 Subject: [PATCH 1/2] fix(callgrind): keep frames below an altstack signal handler Callgrind pops shadow call stack frames whose saved SP is below the current SP. When a signal handler runs on a sigaltstack mapped above the interrupted stack, the first BB of the handler already has a higher SP than every interrupted frame, so the unwind popped through the handler's call_stack_bottom. That looks like a longjmp out of the handler, so post_signal ran early, and the handler's real rt_sigreturn then failed `Assertion 'sigNum == CLG_(current_state).sig'`. Go runs every signal handler on a per-thread sigaltstack above its heap-allocated goroutine stacks, so any Go program aborted under Callgrind on its first preemption signal (SIGURG) or GC. Stop unwinding at the bottom of a handler that runs on the sigaltstack while SP is still inside that altstack. A longjmp that leaves the altstack still unwinds as before. Refs COD-4081 --- CODSPEED-CHANGELOG.md | 8 ++++++++ callgrind/callstack.c | 4 ++++ callgrind/global.h | 4 ++++ callgrind/threads.c | 29 +++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 6416e4bf7..30a90813c 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -11,6 +11,14 @@ last-level cache when `--LL` (or its `--L2` alias) supplies the cache configurat The warning remains when LL is auto-detected. Cache geometry and simulation results are unchanged. +### Callgrind: no crash on signals handled on an alternate stack + +Callgrind no longer aborts with `Assertion 'sigNum == CLG_(current_state).sig' +failed` when a signal handler runs on a `sigaltstack` placed above the +interrupted stack. The stack-pointer based unwinding mistook such a handler for +a `longjmp` out of the handler. Go programs hit this on every preemption signal +(`SIGURG`) and during garbage collection. + ## Features ### Fixed Linux startup stack layout diff --git a/callgrind/callstack.c b/callgrind/callstack.c index d6cedccba..34bc2d18d 100644 --- a/callgrind/callstack.c +++ b/callgrind/callstack.c @@ -482,6 +482,10 @@ Int CLG_(unwind_call_stack)(Addr sp, Int minpops) if ((top_ce->sp < sp) || ((top_ce->sp == sp) && minpops>0)) { + /* Frames below an altstack handler are on another stack */ + if (CLG_(sp_in_altstack_signal_handler)(sp)) + break; + minpops--; unwind_count++; CLG_(pop_call_stack)(); diff --git a/callgrind/global.h b/callgrind/global.h index 97a22bf25..305df418b 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -528,6 +528,9 @@ struct _exec_state { BBCC* nonskipped; Int call_stack_bottom; /* Index into fn_stack */ + + /* The signal handler runs on the thread's sigaltstack */ + Bool on_altstack; }; /* Global state structures */ @@ -834,6 +837,7 @@ void CLG_(set_current_exec_stack)(exec_stack*); void CLG_(pre_signal)(ThreadId tid, Int sigNum, Bool alt_stack); void CLG_(post_signal)(ThreadId tid, Int sigNum); void CLG_(run_post_signal_on_call_stack_bottom)(void); +Bool CLG_(sp_in_altstack_signal_handler)(Addr sp); /* from dump.c */ void CLG_(init_dumps)(void); diff --git a/callgrind/threads.c b/callgrind/threads.c index 642813e4c..9a707d2db 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -336,6 +336,7 @@ void CLG_(pre_signal)(ThreadId tid, Int sigNum, Bool alt_stack) CLG_(zero_cost)( CLG_(sets).full, es->cost ); CLG_(current_state).cost = es->cost; es->call_stack_bottom = CLG_(current_call_stack).sp; + es->on_altstack = alt_stack; /* setup current state for a spontaneous call */ CLG_(init_exec_state)( &CLG_(current_state) ); @@ -358,6 +359,34 @@ void CLG_(run_post_signal_on_call_stack_bottom)(void) CLG_(post_signal)( CLG_(current_tid), CLG_(current_state).sig ); } +/* True if the call stack is at the bottom of a signal handler running on + * the thread's sigaltstack, and `sp` still lies inside that altstack. + * + * The frames below the handler's call_stack_bottom live on the interrupted + * stack, so their SPs cannot be compared with an SP on the altstack: the + * altstack may be mapped above the interrupted stack (e.g. Go's per-M signal + * stacks vs. its heap-allocated goroutine stacks). Without this check the + * SP-based unwind pops through the bottom, which looks like a longjmp out of + * the handler, and the handler's real rt_sigreturn then fails the signal + * number check in post_signal. + */ +Bool CLG_(sp_in_altstack_signal_handler)(Addr sp) +{ + exec_state* es; + Addr alt_min; + SizeT alt_size; + + if (CLG_(current_state).sig <= 0) return False; + + es = top_exec_state(); + if (!es->on_altstack) return False; + if (CLG_(current_call_stack).sp != es->call_stack_bottom) return False; + + alt_min = VG_(thread_get_altstack_min)(CLG_(current_tid)); + alt_size = VG_(thread_get_altstack_size)(CLG_(current_tid)); + return sp >= alt_min && sp - alt_min < alt_size; +} + void CLG_(post_signal)(ThreadId tid, Int sigNum) { exec_state* es; From 756bb4933eb0414e80f928115d8a0406680c46ee Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 30 Oct 2026 20:40:46 +0100 Subject: [PATCH 2/2] feat(callgrind): keep one call stack per Go goroutine on amd64 Go runs many goroutines on one OS thread, each on its own heap-allocated stack, and moves between them without call/return pairs: runtime.gogo jumps into another goroutine, runtime.mcall leaves one for the g0 stack, and runtime.systemstack, runtime.morestack and the vDSO/cgo calls run code on g0 and come back. Callgrind's SP-based shadow stack mixed these stacks, so frames of one goroutine ended up under another, scheduler code under user code, and call graphs of Go programs were unusable. On amd64, track Go stacks explicitly once a Go runtime switch function is seen: - Each goroutine gets its own shadow call stack, keyed by its g pointer (R14). gogo and mcall swap stacks; a goroutine resumed on another thread takes its stack from there. - Frames that run on g0 for a goroutine stay under its caller and are tagged, so the SP comparison never pops frames of the other stack. - Time a goroutine spends suspended is removed from its frames' inclusive cost, active-function counts follow the running goroutine, and shadow frames follow the runtime when it moves a goroutine's stack. - The jump back to a function's entry after morestack is not a call. - Underflow frames on goroutine stacks start at the current cost instead of the last dump. Programs without a Go runtime, and other architectures, take none of these paths. Some call edges inside the Go GC still carry a timing-dependent recursion level. Refs COD-4081 --- CODSPEED-CHANGELOG.md | 14 + callgrind/Makefile.am | 1 + callgrind/bbcc.c | 37 ++- callgrind/callstack.c | 12 +- callgrind/fn.c | 2 + callgrind/global.h | 37 ++- callgrind/gostack.c | 645 ++++++++++++++++++++++++++++++++++++++++++ callgrind/main.c | 5 + callgrind/threads.c | 3 + 9 files changed, 749 insertions(+), 7 deletions(-) create mode 100644 callgrind/gostack.c diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 30a90813c..c781c4ca6 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -21,6 +21,20 @@ a `longjmp` out of the handler. Go programs hit this on every preemption signal ## Features +### Callgrind: Go goroutine call stacks (amd64) + +On amd64, Callgrind keeps one call stack per Go goroutine, so each goroutine's +frames stay under its own entry function without `GODEBUG`, `GOMAXPROCS` or cgo +settings. Callgrind follows the Go runtime's stack switches (`runtime.gogo`, +`runtime.mcall`, `runtime.systemstack`, `runtime.morestack` and the vDSO and cgo +calls that run on the g0 stack), keyed by the goroutine's `g` register. Work a +goroutine does on the g0 stack stays under its caller, time a goroutine spends +suspended is not part of its frames' inclusive cost, and stacks the runtime +moves are followed. + +Known limitation: some call edges inside the Go runtime's garbage collector +still carry a recursion level (`'2`) that depends on GC timing. + ### Fixed Linux startup stack layout Reduce simulated cache-miss variation from argument and environment sizes diff --git a/callgrind/Makefile.am b/callgrind/Makefile.am index de92e16b8..896736ef5 100644 --- a/callgrind/Makefile.am +++ b/callgrind/Makefile.am @@ -50,6 +50,7 @@ CALLGRIND_SOURCES_COMMON = \ dump.c \ events.c \ fn.c \ + gostack.c \ jumps.c \ main.c \ sim.c \ diff --git a/callgrind/bbcc.c b/callgrind/bbcc.c index 910d49419..0c9c040a5 100644 --- a/callgrind/bbcc.c +++ b/callgrind/bbcc.c @@ -545,8 +545,15 @@ static void handleUnderflow(BB* bb) call_entry_up = &(CLG_(current_call_stack).entry[CLG_(current_call_stack).sp -1]); /* assume this call is lasting since last dump or - * for a signal handler since it's call */ - if (CLG_(current_state).sig == 0) + * for a signal handler since it's call. + * A Go goroutine's call stack starts empty when the goroutine starts (or + * when callgrind first sees it), so its caller frames cannot be older than + * now: counting from the last dump would charge them for every goroutine + * that ran on this thread before. */ + if (CLG_(go_runtime)) + CLG_(copy_cost)( CLG_(sets).full, call_entry_up->enter_cost, + CLG_(current_state).cost ); + else if (CLG_(current_state).sig == 0) CLG_(copy_cost)( CLG_(sets).full, call_entry_up->enter_cost, CLG_(get_current_thread)()->lastdump_cost ); else @@ -572,6 +579,7 @@ void CLG_(setup_bbcc)(BB* bb) Int passed = 0, csp; Bool ret_without_call = False; Int popcount_on_return = 1; + Int go_sync = GO_SYNC_NONE; CLG_DEBUG(3,"+ setup_bbcc(BB %#lx)\n", bb_addr(bb)); @@ -650,6 +658,24 @@ void CLG_(setup_bbcc)(BB* bb) isConditionalJump = False; } + /* Go stack switches; a swapped call stack has its own previous BB */ + if (UNLIKELY(CLG_(go_runtime))) { + go_sync = CLG_(go_sync)(bb, last_bb, sp); + if (go_sync == GO_SYNC_SWAPPED) { + last_bbcc = CLG_(current_state).bbcc; + last_bb = last_bbcc ? last_bbcc->bb : 0; + if (last_bb) { + passed = CLG_(current_state).jmps_passed; + jmpkind = last_bb->jmp[passed].jmpkind; + isConditionalJump = (passed < last_bb->cjmp_count); + } + else { + jmpkind = jk_None; + isConditionalJump = False; + } + } + } + /* Manipulate JmpKind if needed, only using BB specific info */ csp = CLG_(current_call_stack).sp; @@ -692,7 +718,8 @@ void CLG_(setup_bbcc)(BB* bb) /* Should this jump be converted to call or pop/call ? */ if (( jmpkind != jk_Return) && - ( jmpkind != jk_Call) && last_bb) { + ( jmpkind != jk_Call) && last_bb && + ( go_sync != GO_SYNC_RESTART)) { /* We simulate a JMP/Cont to be a CALL if * - jump is in another ELF object or section kind @@ -790,11 +817,11 @@ void CLG_(setup_bbcc)(BB* bb) } else { CLG_ASSERT(popcount_on_return >0); - CLG_(unwind_call_stack)(sp, popcount_on_return); + CLG_(unwind_call_stack)(sp, popcount_on_return, bb->fn); } } else { - Int unwind_count = CLG_(unwind_call_stack)(sp, 0); + Int unwind_count = CLG_(unwind_call_stack)(sp, 0, bb->fn); if (unwind_count > 0) { /* if unwinding was done, this actually is a return */ jmpkind = jk_Return; diff --git a/callgrind/callstack.c b/callgrind/callstack.c index 34bc2d18d..7450e7885 100644 --- a/callgrind/callstack.c +++ b/callgrind/callstack.c @@ -296,6 +296,7 @@ void CLG_(push_call_stack)(BBCC* from, UInt jmp, BBCC* to, Addr sp, Bool skip) current_entry->sp = sp; current_entry->ret_addr = ret_addr; current_entry->nonskipped = CLG_(current_state).nonskipped; + current_entry->go_tag = CLG_(go_cur); CLG_(current_call_stack).sp++; @@ -428,6 +429,8 @@ void CLG_(pop_call_stack)(void) CLG_(current_call_stack).sp--; + if (CLG_(go_runtime)) CLG_(go_after_pop)(); + #if CLG_ENABLE_DEBUG CLG_DEBUGIF(1) { if (CLG_(clo).verbose<4) { @@ -461,9 +464,10 @@ void CLG_(pop_call_stack)(void) /* Unwind enough CallStack items to sync with current stack pointer. + * is the function of the BB about to run. * Returns the number of stack frames unwinded. */ -Int CLG_(unwind_call_stack)(Addr sp, Int minpops) +Int CLG_(unwind_call_stack)(Addr sp, Int minpops, fn_node* running) { Int csp; Int unwind_count = 0; @@ -486,6 +490,11 @@ Int CLG_(unwind_call_stack)(Addr sp, Int minpops) if (CLG_(sp_in_altstack_signal_handler)(sp)) break; + /* Frames of another Go stack, see gostack.c */ + if (CLG_(go_runtime) && + CLG_(go_frame_blocks_unwind)(top_ce, running)) + break; + minpops--; unwind_count++; CLG_(pop_call_stack)(); @@ -577,6 +586,7 @@ static void seed_call_stack_from_native(ThreadId tid) call_entry* ce = &cs->entry[cs->sp]; ce->jcc = 0; ce->nonskipped = 0; + ce->go_tag = CLG_(go_cur); /* callgrind pops a frame when SP >= ce->sp, where ce->sp must be the * frame's *entry* SP (the SP at which its caller made the call). The diff --git a/callgrind/fn.c b/callgrind/fn.c index f1d96aca1..b6c83cbfb 100644 --- a/callgrind/fn.c +++ b/callgrind/fn.c @@ -457,6 +457,7 @@ fn_node* new_fn_node(const HChar *fnname, fn->is_malloc = False; fn->is_realloc = False; fn->is_free = False; + fn->go_switch = GO_SWITCH_NONE; fn->group = 0; fn->separate_callers = CLG_(clo).separate_callers; @@ -750,6 +751,7 @@ fn_node* CLG_(get_fn_node)(BB* bb) fn->is_malloc = (VG_(strcmp)(fn->name, "malloc")==0); fn->is_realloc = (VG_(strcmp)(fn->name, "realloc")==0); fn->is_free = (VG_(strcmp)(fn->name, "free")==0); + CLG_(go_classify_fn)(fn); /* apply config options from function name patterns * given on command line */ diff --git a/callgrind/global.h b/callgrind/global.h index 305df418b..2ced49fde 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -41,6 +41,7 @@ #include "pub_tool_options.h" #include "pub_tool_tooliface.h" #include "pub_tool_xarray.h" +#include "pub_tool_hashtable.h" #include "pub_tool_clientstate.h" #include "pub_tool_machine.h" // VG_(fnptr_to_fnentry) @@ -439,6 +440,9 @@ struct _fn_node { Bool is_realloc :1; Bool is_free :1; + /* How this Go runtime function switches stacks, see gostack.c */ + UInt go_switch :3; + Int group; Int separate_callers; Int separate_recursions; @@ -494,6 +498,7 @@ struct _call_entry { BBCC* nonskipped; /* see above */ Context* cxt; /* context before call */ Int fn_sp; /* function stack index before call */ + UWord go_tag; /* Go stack the called frame runs on, 0 if unknown */ }; @@ -630,6 +635,11 @@ struct _thread_info { fn_array fn_active; jcc_hash jccs; bbcc_hash bbccs; + + /* Go goroutine this thread runs, see gostack.c */ + UWord go_owner; + UWord go_cur; + fn_node* go_restart; }; /* Structs used for dumping */ @@ -802,7 +812,7 @@ call_entry* CLG_(get_call_entry)(Int n); void CLG_(push_call_stack)(BBCC* from, UInt jmp, BBCC* to, Addr sp, Bool skip); void CLG_(pop_call_stack)(void); -Int CLG_(unwind_call_stack)(Addr sp, Int); +Int CLG_(unwind_call_stack)(Addr sp, Int, fn_node* running); void CLG_(reconstruct_call_stack_from_native)(ThreadId tid); /* from context.c */ @@ -839,6 +849,31 @@ void CLG_(post_signal)(ThreadId tid, Int sigNum); void CLG_(run_post_signal_on_call_stack_bottom)(void); Bool CLG_(sp_in_altstack_signal_handler)(Addr sp); +/* from gostack.c */ +/* Values of fn_node.go_switch */ +#define GO_SWITCH_NONE 0 +#define GO_SWITCH_GOGO 1 /* runtime.gogo: resume another goroutine */ +#define GO_SWITCH_MCALL 2 /* runtime.mcall: leave goroutine for g0 */ +#define GO_SWITCH_NESTED 3 /* systemstack: run on g0, come back */ +#define GO_SWITCH_MORESTACK 4 /* morestack: grow the stack on g0, restart */ +extern Bool CLG_(go_runtime); +extern UWord CLG_(go_owner); +extern UWord CLG_(go_cur); +extern fn_node* CLG_(go_restart); +void CLG_(go_classify_fn)(fn_node* fn); +Bool CLG_(go_frame_blocks_unwind)(call_entry* ce, fn_node* running); +void CLG_(go_after_pop)(void); +/* Results of CLG_(go_sync) */ +#define GO_SYNC_NONE 0 +#define GO_SYNC_SWAPPED 1 /* another goroutine's call stack is current */ +#define GO_SYNC_RESTART 2 /* jump back to the entry after morestack */ +Int CLG_(go_sync)(BB* bb, BB* last_bb, Addr sp); +void CLG_(go_init_thread)(thread_info* t); +void CLG_(go_save_thread)(thread_info* t); +void CLG_(go_load_thread)(thread_info* t); +void CLG_(go_unwind_thread)(void); +void CLG_(go_unwind_all)(void); + /* from dump.c */ void CLG_(init_dumps)(void); /* Queue a "desc:" line for the header of the next dumped part. */ diff --git a/callgrind/gostack.c b/callgrind/gostack.c new file mode 100644 index 000000000..72a296a00 --- /dev/null +++ b/callgrind/gostack.c @@ -0,0 +1,645 @@ +/*--------------------------------------------------------------------*/ +/*--- Callgrind ---*/ +/*--- gostack.c ---*/ +/*--------------------------------------------------------------------*/ + +/* + This file is part of Callgrind, a Valgrind tool for call tracing. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + The GNU General Public License is contained in the file COPYING. +*/ + +/* Go goroutine stacks. + * + * Callgrind keeps one shadow call stack per thread and pops frames by + * comparing stack pointers. The Go runtime runs many goroutines on one + * thread, each on its own heap-allocated stack, and moves between them + * without call/return pairs: + * + * - runtime.gogo jumps into another goroutine (or a new one). + * - runtime.mcall leaves a goroutine for the thread's g0 stack, where the + * scheduler picks the next goroutine. + * - runtime.systemstack and runtime.morestack run a function on g0 and + * then come back to the calling goroutine. + * + * Comparing SPs across these stacks pops unrelated frames, and frames of one + * goroutine end up under another one. So for Go programs each goroutine gets + * its own shadow call stack, keyed by its g pointer (held in a fixed register + * by the Go ABI). gogo and mcall swap shadow stacks. systemstack and + * morestack keep the goroutine's shadow stack, so work done on g0 for the + * goroutine stays under its caller, and tag the frames that run on g0 so the + * SP comparison never pops a goroutine frame from g0, or the other way round. + * + * Only the functions above are looked at. They are found by name, so this + * costs nothing for programs without a Go runtime. + * + * Only amd64 is supported for now. Other architectures keep the plain + * SP-based shadow stack. + */ + +#include "global.h" +#include "pub_tool_threadstate.h" +#include "pub_tool_guest.h" + +#if defined(VGA_amd64) +/* ABIInternal keeps the current g in R14 */ +# define GO_G_REG guest_R14 +/* Bytes a call pushes: the return address */ +# define GO_CALL_PUSH 8 +#endif + +/* Tag of frames that run on g0 for a goroutine (systemstack, morestack). + * Never a valid g pointer. */ +#define GO_TAG_SYSTEM ((UWord)1) + +/* Initial entries of a new goroutine's shadow stacks: goroutines are many and + * mostly shallow, the stacks grow on demand. */ +#define GO_STACK_INITIAL_ENTRIES 16 + +Bool CLG_(go_runtime) = False; +/* g whose shadow stack is the current call stack, 0 if not known yet */ +UWord CLG_(go_owner) = 0; +/* Tag of the stack the running frame is on */ +UWord CLG_(go_cur) = 0; +/* Function that jumps back to its own entry after morestack, see resume() */ +fn_node* CLG_(go_restart) = 0; + +/* A suspended goroutine's shadow stack. The first two fields are the + * VgHashNode header. */ +typedef struct _go_stack go_stack; +struct _go_stack { + go_stack* next; + UWord key; /* g pointer */ + + call_stack calls; + fn_stack fns; + Context* cxt; + BBCC* bbcc; + BBCC* nonskipped; + Int jmps_passed; + FullCost suspend_cost; /* cost counters when the goroutine was left */ + fn_node* root; /* function marked active again on resume */ +}; + +#if defined(GO_G_REG) +typedef struct { + const HChar* name; + UInt kind; +} go_switch_fn; + +/* Names with and without the ".abi0" suffix Go 1.17 added to assembly + * functions. "gogo" is the static target runtime.gogo.abi0 jumps to. */ +static const go_switch_fn go_switch_fns[] = { + { "runtime.gogo.abi0", GO_SWITCH_GOGO }, + { "runtime.gogo", GO_SWITCH_GOGO }, + { "gogo", GO_SWITCH_GOGO }, + { "runtime.mcall", GO_SWITCH_MCALL }, + { "runtime.systemstack.abi0", GO_SWITCH_NESTED }, + { "runtime.systemstack", GO_SWITCH_NESTED }, + { "runtime.morestack.abi0", GO_SWITCH_MORESTACK }, + { "runtime.morestack", GO_SWITCH_MORESTACK }, + /* vDSO calls run on g0 */ + { "runtime.nanotime1.abi0", GO_SWITCH_NESTED }, + { "runtime.nanotime1", GO_SWITCH_NESTED }, + { "runtime.walltime.abi0", GO_SWITCH_NESTED }, + { "runtime.walltime", GO_SWITCH_NESTED }, + { "runtime.walltime1.abi0", GO_SWITCH_NESTED }, + { "runtime.walltime1", GO_SWITCH_NESTED }, + { "runtime.vgetrandom1", GO_SWITCH_NESTED }, + /* cgo calls run on g0 */ + { "runtime.asmcgocall.abi0", GO_SWITCH_NESTED }, + { "runtime.asmcgocall", GO_SWITCH_NESTED }, +}; +#endif + +void CLG_(go_classify_fn)(fn_node* fn) +{ +#if defined(GO_G_REG) + UInt i; + + for (i = 0; i < sizeof(go_switch_fns) / sizeof(go_switch_fns[0]); i++) { + if (VG_(strcmp)(fn->name, go_switch_fns[i].name) != 0) continue; + + /* A plain "gogo" only counts once the Go runtime was seen; its only + * caller, runtime.gogo.abi0, always runs first. */ + if (!CLG_(go_runtime) && fn->name[0] == 'g') return; + + fn->go_switch = go_switch_fns[i].kind; + CLG_(go_runtime) = True; + return; + } +#endif +} + +static fn_node* callee_fn(call_entry* ce) +{ + return ce->jcc ? ce->jcc->to->cxt->fn[0] : 0; +} + +#if defined(GO_G_REG) + +static UWord read_g(void) +{ + UWord g; + VG_(get_shadow_regs_area)(CLG_(current_tid), (UChar*)&g, 0, + offsetof(VexGuestArchState, GO_G_REG), + sizeof(g)); + return g; +} + +/* Tag of the frame that runs after the top frame returns */ +static UWord tag_below_top(void) +{ + Int csp = CLG_(current_call_stack).sp; + UWord tag = csp > 0 ? CLG_(current_call_stack).entry[csp-1].go_tag : 0; + return tag ? tag : CLG_(go_owner); +} + +/* Add to the active counts of the functions on the current call stack, + * the inverse of what push/pop_call_stack do: a suspended goroutine's frames + * must not count as recursion of the running one. */ +static void adjust_active(Int dir) +{ + Int i; + + for (i = 0; i < CLG_(current_call_stack).sp; i++) { + call_entry* ce = &CLG_(current_call_stack).entry[i]; + fn_node* to_fn = callee_fn(ce); + UInt* pdepth; + + if (!to_fn) continue; + if (CLG_(clo).skip_direct_recursion && + ce->jcc->from->cxt->fn[0] == to_fn) continue; + + pdepth = CLG_(get_fn_entry)(to_fn->number); + if (dir < 0 && *pdepth == 0) continue; + *pdepth += dir; + } +} + +/* The function a call stack started in has no call entry, but callgrind marks + * it active (see the recursion level in setup_bbcc). Clear that mark when the + * stack is suspended; returns the function to mark again on resume. */ +static fn_node* uncount_root(void) +{ + Context* cxt = CLG_(current_call_stack).sp > 0 + ? CLG_(current_call_stack).entry[0].cxt + : CLG_(current_state).cxt; + fn_node* fn = cxt ? cxt->fn[0] : 0; + UInt* pdepth; + + if (!fn) return 0; + pdepth = CLG_(get_fn_entry)(fn->number); + if (*pdepth == 0) return 0; + (*pdepth)--; + return fn; +} + +/* Unused call stack arrays, kept in go_stack nodes linked by . Cost + * arrays hanging off call entries come from a pool and cannot be freed, so + * the arrays of finished goroutines are reused instead of freed. */ +static go_stack* spare_stacks = 0; + +/* Suspended goroutines by g pointer. Global: goroutines move between + * threads. */ +static VgHashTable* go_stacks = 0; + +/* Move the current call stack and context into a new go_stack. */ +static go_stack* stack_out(UWord key) +{ + go_stack* s = (go_stack*) CLG_MALLOC("cl.gostack.so.1", sizeof(go_stack)); + + adjust_active(-1); + s->root = uncount_root(); + + s->key = key; + CLG_(copy_current_call_stack)(&s->calls); + CLG_(copy_current_fn_stack)(&s->fns); + s->cxt = CLG_(current_state).cxt; + s->bbcc = CLG_(current_state).bbcc; + s->nonskipped = CLG_(current_state).nonskipped; + s->jmps_passed = CLG_(current_state).jmps_passed; + s->suspend_cost = (FullCost) CLG_MALLOC("cl.gostack.so.2", + CLG_(sets).full->size * sizeof(ULong)); + CLG_(copy_cost)(CLG_(sets).full, s->suspend_cost, CLG_(current_state).cost); + return s; +} + +/* Make 's call stack and context current and free . */ +static void stack_in(go_stack* s) +{ + Int i; + + CLG_(set_current_call_stack)(&s->calls); + CLG_(set_current_fn_stack)(&s->fns); + CLG_(current_state).cxt = s->cxt; + CLG_(current_state).bbcc = s->bbcc; + CLG_(current_state).nonskipped = s->nonskipped; + CLG_(current_state).jmps_passed = s->jmps_passed; + + /* The time the goroutine was suspended is not part of its frames' + * inclusive cost. Not CLG_(add_diff_cost): it moves to . */ + for (i = 0; i < CLG_(current_call_stack).sp; i++) { + call_entry* ce = &CLG_(current_call_stack).entry[i]; + Int e; + if (!ce->jcc || !ce->enter_cost) continue; + for (e = 0; e < CLG_(sets).full->size; e++) + ce->enter_cost[e] += CLG_(current_state).cost[e] - s->suspend_cost[e]; + } + + adjust_active(1); + if (s->root) (*CLG_(get_fn_entry)(s->root->number))++; + + VG_(free)(s->suspend_cost); + VG_(free)(s); +} + +/* Install an empty call stack for a goroutine run for the first time. */ +static void stack_fresh(void) +{ + go_stack* spare = spare_stacks; + call_stack cs; + fn_stack fs; + UInt i; + + if (spare) { + spare_stacks = spare->next; + cs = spare->calls; + fs = spare->fns; + VG_(free)(spare); + } + else { + cs.size = GO_STACK_INITIAL_ENTRIES; + cs.entry = (call_entry*) CLG_MALLOC("cl.gostack.sf.1", + cs.size * sizeof(call_entry)); + for (i = 0; i < cs.size; i++) cs.entry[i].enter_cost = 0; + + fs.size = GO_STACK_INITIAL_ENTRIES; + fs.bottom = (fn_node**) CLG_MALLOC("cl.gostack.sf.2", + fs.size * sizeof(fn_node*)); + } + cs.sp = 0; + cs.entry[0].cxt = 0; + fs.top = fs.bottom; + fs.bottom[0] = 0; + + CLG_(set_current_call_stack)(&cs); + CLG_(set_current_fn_stack)(&fs); + CLG_(current_state).cxt = 0; + CLG_(current_state).bbcc = 0; + CLG_(current_state).nonskipped = 0; + CLG_(current_state).jmps_passed = 0; +} + +/* Keep the arrays of for reuse and free the rest of it, without popping + * its frames: their goroutine is gone. */ +static void stack_free(go_stack* s) +{ + VG_(free)(s->suspend_cost); + s->suspend_cost = 0; + s->next = spare_stacks; + spare_stacks = s; +} + +/* Pop all frames of the current call stack and keep its arrays for reuse. + * A call stack must be installed again afterwards. */ +static void drop_current_stack(void) +{ + go_stack* s = (go_stack*) CLG_MALLOC("cl.gostack.dcs.1", sizeof(go_stack)); + + while (CLG_(current_call_stack).sp > 0) + CLG_(pop_call_stack)(); + uncount_root(); + + CLG_(copy_current_call_stack)(&s->calls); + CLG_(copy_current_fn_stack)(&s->fns); + s->suspend_cost = 0; + s->next = spare_stacks; + spare_stacks = s; +} + +/* A goroutine stops at a call into mcall or morestack and is resumed with + * SP where that call left it. The Go runtime moves goroutine stacks when + * they grow or shrink, so shift the shadow frames by the same distance. */ +static void relocate(Addr sp) +{ + Int csp = CLG_(current_call_stack).sp; + call_entry* top; + fn_node* fn; + Addr expected; + Int i; + + if (csp == 0) return; + top = &CLG_(current_call_stack).entry[csp-1]; + fn = callee_fn(top); + if (!fn || fn->go_switch == GO_SWITCH_NONE) return; + + expected = top->sp + GO_CALL_PUSH; + if (sp == expected) return; + + for (i = 0; i < csp; i++) + CLG_(current_call_stack).entry[i].sp += sp - expected; +} + +/* Continue the current goroutine at , where it stopped in mcall or + * morestack. A function that ran out of stack calls morestack from its + * prologue and, once the stack has grown, jumps back to its own entry. Mark + * that function so the jump does not count as a call of itself. */ +static void resume(Addr sp) +{ + static const HChar morestack_prefix[] = "runtime.morestack"; + fn_node* fn; + Int i; + + relocate(sp); + + i = CLG_(current_call_stack).sp - 1; + if (i < 0) return; + fn = callee_fn(&CLG_(current_call_stack).entry[i]); + if (!fn || fn->go_switch != GO_SWITCH_MORESTACK) return; + + /* Below morestack, and morestack_noctxt jumping to it */ + for (; i >= 0; i--) { + fn = callee_fn(&CLG_(current_call_stack).entry[i]); + if (!fn || VG_(strncmp)(fn->name, morestack_prefix, + sizeof(morestack_prefix) - 1) != 0) break; + } + CLG_(go_restart) = i >= 0 ? fn : 0; +} + +/* Pop the frames on top that run on g0 for the current goroutine. */ +static void pop_nested_frames(void) +{ + for (;;) { + Int csp = CLG_(current_call_stack).sp; + UWord tag; + if (csp == 0) return; + tag = CLG_(current_call_stack).entry[csp-1].go_tag; + if (tag == 0 || tag == CLG_(go_owner)) return; + CLG_(pop_call_stack)(); + } +} + +/* Suspend the current call stack under its goroutine. If the goroutine is + * not known (no switch function was entered yet), its frames are popped. + * + * The scheduler may run inside a nested g0 call (newstack preempting the + * goroutine), so pop those g0 frames: the goroutine resumes below them. */ +static void leave_current(void) +{ + if (!go_stacks) go_stacks = VG_(HT_construct)("cl.gostack.lc.1"); + + if (CLG_(go_owner)) { + pop_nested_frames(); + VG_(HT_add_node)(go_stacks, stack_out(CLG_(go_owner))); + return; + } + drop_current_stack(); +} + +/* A goroutine can be descheduled while it runs on g0 for itself, without + * mcall: newstack preempts it and the scheduler finds nothing else to run, + * so the thread sleeps with the goroutine's call stack still current. Take + * that call stack when another thread resumes the goroutine. */ +static go_stack* steal_from_other_thread(UWord g) +{ + thread_info** threads = CLG_(get_threads)(); + ThreadId me = CLG_(current_tid); + ThreadId tid; + go_stack* s; + + for (tid = 1; tid < VG_N_THREADS; tid++) { + thread_info* t = threads[tid]; + if (!t || tid == me || t->go_owner != g) continue; + + CLG_(switch_thread)(tid); + if (CLG_(current_state).sig > 0) { + /* Not reachable below a signal handler; forget the goroutine */ + CLG_(go_owner) = CLG_(go_cur) = 0; + CLG_(switch_thread)(me); + return 0; + } + pop_nested_frames(); + s = stack_out(g); + stack_fresh(); + CLG_(go_owner) = CLG_(go_cur) = 0; + CLG_(switch_thread)(me); + return s; + } + return 0; +} + +/* gogo jumps to the goroutine , starting it if is a function entry. */ +static Bool gogo_to(UWord g, BB* bb, Addr sp) +{ + go_stack* s; + + if (g == 0 || g == GO_TAG_SYSTEM) return False; + + if (g == CLG_(go_owner)) { + /* Back to the goroutine whose stack we are on, after it ran on g0 + * (morestack -> newstack -> gogo). */ + pop_nested_frames(); + resume(sp); + CLG_(go_cur) = tag_below_top(); + return False; + } + + leave_current(); + + s = VG_(HT_remove)(go_stacks, g); + if (!s) s = steal_from_other_thread(g); + if (s && bb->is_entry) { + /* g was reused for a new goroutine; the old one exited */ + stack_free(s); + s = 0; + } + + if (s) { + stack_in(s); + resume(sp); + } + else + stack_fresh(); + + CLG_(go_owner) = g; + CLG_(go_cur) = tag_below_top(); + return True; +} + +/* mcall runs a function on g0 that never returns; g0 starts over from the + * bottom of its stack. */ +static void mcall_to(UWord g0) +{ + go_stack* s; + + if (g0 == 0 || g0 == GO_TAG_SYSTEM || g0 == CLG_(go_owner)) return; + + leave_current(); + + /* Pop what g0 was doing when it last left (the scheduler up to gogo) */ + s = VG_(HT_remove)(go_stacks, g0); + if (s) { + stack_in(s); + drop_current_stack(); + } + stack_fresh(); + + CLG_(go_owner) = g0; + CLG_(go_cur) = g0; +} + +#endif /* GO_G_REG */ + +Bool CLG_(go_frame_blocks_unwind)(call_entry* ce, fn_node* running) +{ + fn_node* fn; + + /* Still inside the switch function: its SP is on either stack */ + fn = callee_fn(ce); + if (fn && fn == running && fn->go_switch != GO_SWITCH_NONE) return True; + + /* A frame of another stack */ + return ce->go_tag && CLG_(go_cur) && ce->go_tag != CLG_(go_cur); +} + +void CLG_(go_after_pop)(void) +{ +#if defined(GO_G_REG) + CLG_(go_cur) = tag_below_top(); +#endif +} + +/* Called for every BB of a Go program before the jump into is + * classified. Returns GO_SYNC_SWAPPED if the current call stack was swapped + * for another goroutine's, so the caller must reload its view of the previous + * BB, and GO_SYNC_RESTART if the jump into must not become a call. */ +Int CLG_(go_sync)(BB* bb, BB* last_bb, Addr sp) +{ +#if defined(GO_G_REG) + fn_node* fn = bb->fn; + fn_node* last_fn = last_bb ? last_bb->fn : 0; + fn_node* restart = CLG_(go_restart); + ClgJumpKind jmpkind; + + /* Go never switches goroutines inside a signal handler */ + if (CLG_(current_state).sig > 0) return GO_SYNC_NONE; + + if (restart) { + /* Still in the restarting function until it jumps to its entry */ + if (fn != restart || bb->is_entry) CLG_(go_restart) = 0; + if (fn == restart && bb->is_entry && last_fn == restart) + return GO_SYNC_RESTART; + } + + if (fn->go_switch != GO_SWITCH_NONE && fn != last_fn) { + /* Learn which goroutine we are on: the g register is valid when a + * switch function is entered */ + if (CLG_(go_owner) == 0) + CLG_(go_owner) = CLG_(go_cur) = read_g(); + return GO_SYNC_NONE; + } + + if (!last_fn || last_fn->go_switch == GO_SWITCH_NONE || fn == last_fn) + return GO_SYNC_NONE; + + jmpkind = last_bb->jmp[CLG_(current_state).jmps_passed].jmpkind; + if (jmpkind == jk_Return) return GO_SYNC_NONE; + + switch (last_fn->go_switch) { + case GO_SWITCH_GOGO: + return gogo_to(read_g(), bb, sp) ? GO_SYNC_SWAPPED : GO_SYNC_NONE; + case GO_SWITCH_MCALL: + mcall_to(read_g()); + return GO_SYNC_SWAPPED; + case GO_SWITCH_NESTED: + case GO_SWITCH_MORESTACK: + /* A call into the function to run on g0. Returning from it is handled + * by the pop. */ + CLG_(go_cur) = GO_TAG_SYSTEM; + return GO_SYNC_NONE; + default: + return GO_SYNC_NONE; + } +#else + return GO_SYNC_NONE; +#endif +} + +void CLG_(go_init_thread)(thread_info* t) +{ + t->go_owner = 0; + t->go_cur = 0; + t->go_restart = 0; +} + +void CLG_(go_save_thread)(thread_info* t) +{ + t->go_owner = CLG_(go_owner); + t->go_cur = CLG_(go_cur); + t->go_restart = CLG_(go_restart); +} + +void CLG_(go_load_thread)(thread_info* t) +{ + CLG_(go_owner) = t->go_owner; + CLG_(go_cur) = t->go_cur; + CLG_(go_restart) = t->go_restart; +} + +/* The current thread's call stack was popped: start over without a known + * goroutine. */ +void CLG_(go_unwind_thread)(void) +{ + CLG_(go_owner) = 0; + CLG_(go_cur) = 0; + CLG_(go_restart) = 0; +} + +/* Pop the frames of all suspended goroutines. */ +void CLG_(go_unwind_all)(void) +{ +#if defined(GO_G_REG) + call_stack cs; + fn_stack fs; + exec_state saved; + go_stack* s; + UInt i, n; + UWord* keys; + + if (!go_stacks || VG_(HT_count_nodes)(go_stacks) == 0) return; + + CLG_(copy_current_call_stack)(&cs); + CLG_(copy_current_fn_stack)(&fs); + saved = CLG_(current_state); + + n = VG_(HT_count_nodes)(go_stacks); + keys = (UWord*) CLG_MALLOC("cl.gostack.ua.1", n * sizeof(UWord)); + VG_(HT_ResetIter)(go_stacks); + for (i = 0; i < n; i++) keys[i] = ((go_stack*)VG_(HT_Next)(go_stacks))->key; + + for (i = 0; i < n; i++) { + s = VG_(HT_remove)(go_stacks, keys[i]); + stack_in(s); + drop_current_stack(); + } + VG_(free)(keys); + + CLG_(set_current_call_stack)(&cs); + CLG_(set_current_fn_stack)(&fs); + CLG_(current_state) = saved; +#endif +} diff --git a/callgrind/main.c b/callgrind/main.c index 69af08255..53fc3aa0d 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -1463,6 +1463,9 @@ void CLG_(unwind_thread)(thread_info* t) while(CLG_(current_call_stack).sp>0) CLG_(pop_call_stack)(); + /* no goroutine is known to run on the empty stack */ + CLG_(go_unwind_thread)(); + /* reset context and function stack for context generation */ CLG_(init_exec_state)( &CLG_(current_state) ); CLG_(current_fn_stack).top = CLG_(current_fn_stack).bottom; @@ -1492,6 +1495,7 @@ void CLG_(set_instrument_state)(const HChar* reason, Bool state) /* reset internal state: call stacks, simulator */ CLG_(forall_threads)(CLG_(unwind_thread)); + CLG_(go_unwind_all)(); CLG_(forall_threads)(zero_state_cost); (*CLG_(cachesim).clear)(); @@ -2016,6 +2020,7 @@ void finish(void) /* pop all remaining items from CallStack for correct sum */ CLG_(forall_threads)(CLG_(unwind_thread)); + CLG_(go_unwind_all)(); CLG_(dump_profile)(0, False); diff --git a/callgrind/threads.c b/callgrind/threads.c index 9a707d2db..bfe5f5deb 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -238,6 +238,7 @@ thread_info* new_thread(ThreadId tid) CLG_(init_fn_array)( &(t->fn_active) ); CLG_(init_bbcc_hash)( &(t->bbccs) ); CLG_(init_jcc_hash)( &(t->jccs) ); + CLG_(go_init_thread)(t); return t; } @@ -255,6 +256,7 @@ static void load_thread_state(thread_info* t) CLG_(set_current_fn_stack) ( &(t->fns) ); CLG_(set_current_fn_array) ( &(t->fn_active) ); + CLG_(go_load_thread)(t); CLG_(current_thread_serial) = t->serial; /* If we cumulate costs of threads, use TID 1 for all jccs/bccs */ if (!CLG_(clo).separate_threads) t = thread[1]; @@ -282,6 +284,7 @@ void CLG_(switch_thread)(ThreadId tid) CLG_(copy_current_fn_stack) ( &(t->fns) ); CLG_(copy_current_fn_array) ( &(t->fn_active) ); + CLG_(go_save_thread)(t); /* If we cumulate costs of threads, use TID 1 for all jccs/bccs */ if (!CLG_(clo).separate_threads) t = thread[1]; CLG_(copy_current_bbcc_hash)( &(t->bbccs) );