diff --git a/CODSPEED-CHANGELOG.md b/CODSPEED-CHANGELOG.md index 6416e4bf7..c781c4ca6 100644 --- a/CODSPEED-CHANGELOG.md +++ b/CODSPEED-CHANGELOG.md @@ -11,8 +11,30 @@ last-level cache when `--LL` (or its `--L2` alias) supplies the cache configurat The warning remains when LL is auto-detected. Cache geometry and simulation results are unchanged. +### Callgrind: no crash on signals handled on an alternate stack + +Callgrind no longer aborts with `Assertion 'sigNum == CLG_(current_state).sig' +failed` when a signal handler runs on a `sigaltstack` placed above the +interrupted stack. The stack-pointer based unwinding mistook such a handler for +a `longjmp` out of the handler. Go programs hit this on every preemption signal +(`SIGURG`) and during garbage collection. + ## Features +### Callgrind: Go goroutine call stacks (amd64) + +On amd64, Callgrind keeps one call stack per Go goroutine, so each goroutine's +frames stay under its own entry function without `GODEBUG`, `GOMAXPROCS` or cgo +settings. Callgrind follows the Go runtime's stack switches (`runtime.gogo`, +`runtime.mcall`, `runtime.systemstack`, `runtime.morestack` and the vDSO and cgo +calls that run on the g0 stack), keyed by the goroutine's `g` register. Work a +goroutine does on the g0 stack stays under its caller, time a goroutine spends +suspended is not part of its frames' inclusive cost, and stacks the runtime +moves are followed. + +Known limitation: some call edges inside the Go runtime's garbage collector +still carry a recursion level (`'2`) that depends on GC timing. + ### Fixed Linux startup stack layout Reduce simulated cache-miss variation from argument and environment sizes diff --git a/callgrind/Makefile.am b/callgrind/Makefile.am index de92e16b8..896736ef5 100644 --- a/callgrind/Makefile.am +++ b/callgrind/Makefile.am @@ -50,6 +50,7 @@ CALLGRIND_SOURCES_COMMON = \ dump.c \ events.c \ fn.c \ + gostack.c \ jumps.c \ main.c \ sim.c \ diff --git a/callgrind/bbcc.c b/callgrind/bbcc.c index 910d49419..0c9c040a5 100644 --- a/callgrind/bbcc.c +++ b/callgrind/bbcc.c @@ -545,8 +545,15 @@ static void handleUnderflow(BB* bb) call_entry_up = &(CLG_(current_call_stack).entry[CLG_(current_call_stack).sp -1]); /* assume this call is lasting since last dump or - * for a signal handler since it's call */ - if (CLG_(current_state).sig == 0) + * for a signal handler since it's call. + * A Go goroutine's call stack starts empty when the goroutine starts (or + * when callgrind first sees it), so its caller frames cannot be older than + * now: counting from the last dump would charge them for every goroutine + * that ran on this thread before. */ + if (CLG_(go_runtime)) + CLG_(copy_cost)( CLG_(sets).full, call_entry_up->enter_cost, + CLG_(current_state).cost ); + else if (CLG_(current_state).sig == 0) CLG_(copy_cost)( CLG_(sets).full, call_entry_up->enter_cost, CLG_(get_current_thread)()->lastdump_cost ); else @@ -572,6 +579,7 @@ void CLG_(setup_bbcc)(BB* bb) Int passed = 0, csp; Bool ret_without_call = False; Int popcount_on_return = 1; + Int go_sync = GO_SYNC_NONE; CLG_DEBUG(3,"+ setup_bbcc(BB %#lx)\n", bb_addr(bb)); @@ -650,6 +658,24 @@ void CLG_(setup_bbcc)(BB* bb) isConditionalJump = False; } + /* Go stack switches; a swapped call stack has its own previous BB */ + if (UNLIKELY(CLG_(go_runtime))) { + go_sync = CLG_(go_sync)(bb, last_bb, sp); + if (go_sync == GO_SYNC_SWAPPED) { + last_bbcc = CLG_(current_state).bbcc; + last_bb = last_bbcc ? last_bbcc->bb : 0; + if (last_bb) { + passed = CLG_(current_state).jmps_passed; + jmpkind = last_bb->jmp[passed].jmpkind; + isConditionalJump = (passed < last_bb->cjmp_count); + } + else { + jmpkind = jk_None; + isConditionalJump = False; + } + } + } + /* Manipulate JmpKind if needed, only using BB specific info */ csp = CLG_(current_call_stack).sp; @@ -692,7 +718,8 @@ void CLG_(setup_bbcc)(BB* bb) /* Should this jump be converted to call or pop/call ? */ if (( jmpkind != jk_Return) && - ( jmpkind != jk_Call) && last_bb) { + ( jmpkind != jk_Call) && last_bb && + ( go_sync != GO_SYNC_RESTART)) { /* We simulate a JMP/Cont to be a CALL if * - jump is in another ELF object or section kind @@ -790,11 +817,11 @@ void CLG_(setup_bbcc)(BB* bb) } else { CLG_ASSERT(popcount_on_return >0); - CLG_(unwind_call_stack)(sp, popcount_on_return); + CLG_(unwind_call_stack)(sp, popcount_on_return, bb->fn); } } else { - Int unwind_count = CLG_(unwind_call_stack)(sp, 0); + Int unwind_count = CLG_(unwind_call_stack)(sp, 0, bb->fn); if (unwind_count > 0) { /* if unwinding was done, this actually is a return */ jmpkind = jk_Return; diff --git a/callgrind/callstack.c b/callgrind/callstack.c index d6cedccba..7450e7885 100644 --- a/callgrind/callstack.c +++ b/callgrind/callstack.c @@ -296,6 +296,7 @@ void CLG_(push_call_stack)(BBCC* from, UInt jmp, BBCC* to, Addr sp, Bool skip) current_entry->sp = sp; current_entry->ret_addr = ret_addr; current_entry->nonskipped = CLG_(current_state).nonskipped; + current_entry->go_tag = CLG_(go_cur); CLG_(current_call_stack).sp++; @@ -428,6 +429,8 @@ void CLG_(pop_call_stack)(void) CLG_(current_call_stack).sp--; + if (CLG_(go_runtime)) CLG_(go_after_pop)(); + #if CLG_ENABLE_DEBUG CLG_DEBUGIF(1) { if (CLG_(clo).verbose<4) { @@ -461,9 +464,10 @@ void CLG_(pop_call_stack)(void) /* Unwind enough CallStack items to sync with current stack pointer. + * is the function of the BB about to run. * Returns the number of stack frames unwinded. */ -Int CLG_(unwind_call_stack)(Addr sp, Int minpops) +Int CLG_(unwind_call_stack)(Addr sp, Int minpops, fn_node* running) { Int csp; Int unwind_count = 0; @@ -482,6 +486,15 @@ Int CLG_(unwind_call_stack)(Addr sp, Int minpops) if ((top_ce->sp < sp) || ((top_ce->sp == sp) && minpops>0)) { + /* Frames below an altstack handler are on another stack */ + if (CLG_(sp_in_altstack_signal_handler)(sp)) + break; + + /* Frames of another Go stack, see gostack.c */ + if (CLG_(go_runtime) && + CLG_(go_frame_blocks_unwind)(top_ce, running)) + break; + minpops--; unwind_count++; CLG_(pop_call_stack)(); @@ -573,6 +586,7 @@ static void seed_call_stack_from_native(ThreadId tid) call_entry* ce = &cs->entry[cs->sp]; ce->jcc = 0; ce->nonskipped = 0; + ce->go_tag = CLG_(go_cur); /* callgrind pops a frame when SP >= ce->sp, where ce->sp must be the * frame's *entry* SP (the SP at which its caller made the call). The diff --git a/callgrind/fn.c b/callgrind/fn.c index f1d96aca1..b6c83cbfb 100644 --- a/callgrind/fn.c +++ b/callgrind/fn.c @@ -457,6 +457,7 @@ fn_node* new_fn_node(const HChar *fnname, fn->is_malloc = False; fn->is_realloc = False; fn->is_free = False; + fn->go_switch = GO_SWITCH_NONE; fn->group = 0; fn->separate_callers = CLG_(clo).separate_callers; @@ -750,6 +751,7 @@ fn_node* CLG_(get_fn_node)(BB* bb) fn->is_malloc = (VG_(strcmp)(fn->name, "malloc")==0); fn->is_realloc = (VG_(strcmp)(fn->name, "realloc")==0); fn->is_free = (VG_(strcmp)(fn->name, "free")==0); + CLG_(go_classify_fn)(fn); /* apply config options from function name patterns * given on command line */ diff --git a/callgrind/global.h b/callgrind/global.h index 97a22bf25..2ced49fde 100644 --- a/callgrind/global.h +++ b/callgrind/global.h @@ -41,6 +41,7 @@ #include "pub_tool_options.h" #include "pub_tool_tooliface.h" #include "pub_tool_xarray.h" +#include "pub_tool_hashtable.h" #include "pub_tool_clientstate.h" #include "pub_tool_machine.h" // VG_(fnptr_to_fnentry) @@ -439,6 +440,9 @@ struct _fn_node { Bool is_realloc :1; Bool is_free :1; + /* How this Go runtime function switches stacks, see gostack.c */ + UInt go_switch :3; + Int group; Int separate_callers; Int separate_recursions; @@ -494,6 +498,7 @@ struct _call_entry { BBCC* nonskipped; /* see above */ Context* cxt; /* context before call */ Int fn_sp; /* function stack index before call */ + UWord go_tag; /* Go stack the called frame runs on, 0 if unknown */ }; @@ -528,6 +533,9 @@ struct _exec_state { BBCC* nonskipped; Int call_stack_bottom; /* Index into fn_stack */ + + /* The signal handler runs on the thread's sigaltstack */ + Bool on_altstack; }; /* Global state structures */ @@ -627,6 +635,11 @@ struct _thread_info { fn_array fn_active; jcc_hash jccs; bbcc_hash bbccs; + + /* Go goroutine this thread runs, see gostack.c */ + UWord go_owner; + UWord go_cur; + fn_node* go_restart; }; /* Structs used for dumping */ @@ -799,7 +812,7 @@ call_entry* CLG_(get_call_entry)(Int n); void CLG_(push_call_stack)(BBCC* from, UInt jmp, BBCC* to, Addr sp, Bool skip); void CLG_(pop_call_stack)(void); -Int CLG_(unwind_call_stack)(Addr sp, Int); +Int CLG_(unwind_call_stack)(Addr sp, Int, fn_node* running); void CLG_(reconstruct_call_stack_from_native)(ThreadId tid); /* from context.c */ @@ -834,6 +847,32 @@ void CLG_(set_current_exec_stack)(exec_stack*); void CLG_(pre_signal)(ThreadId tid, Int sigNum, Bool alt_stack); void CLG_(post_signal)(ThreadId tid, Int sigNum); void CLG_(run_post_signal_on_call_stack_bottom)(void); +Bool CLG_(sp_in_altstack_signal_handler)(Addr sp); + +/* from gostack.c */ +/* Values of fn_node.go_switch */ +#define GO_SWITCH_NONE 0 +#define GO_SWITCH_GOGO 1 /* runtime.gogo: resume another goroutine */ +#define GO_SWITCH_MCALL 2 /* runtime.mcall: leave goroutine for g0 */ +#define GO_SWITCH_NESTED 3 /* systemstack: run on g0, come back */ +#define GO_SWITCH_MORESTACK 4 /* morestack: grow the stack on g0, restart */ +extern Bool CLG_(go_runtime); +extern UWord CLG_(go_owner); +extern UWord CLG_(go_cur); +extern fn_node* CLG_(go_restart); +void CLG_(go_classify_fn)(fn_node* fn); +Bool CLG_(go_frame_blocks_unwind)(call_entry* ce, fn_node* running); +void CLG_(go_after_pop)(void); +/* Results of CLG_(go_sync) */ +#define GO_SYNC_NONE 0 +#define GO_SYNC_SWAPPED 1 /* another goroutine's call stack is current */ +#define GO_SYNC_RESTART 2 /* jump back to the entry after morestack */ +Int CLG_(go_sync)(BB* bb, BB* last_bb, Addr sp); +void CLG_(go_init_thread)(thread_info* t); +void CLG_(go_save_thread)(thread_info* t); +void CLG_(go_load_thread)(thread_info* t); +void CLG_(go_unwind_thread)(void); +void CLG_(go_unwind_all)(void); /* from dump.c */ void CLG_(init_dumps)(void); diff --git a/callgrind/gostack.c b/callgrind/gostack.c new file mode 100644 index 000000000..72a296a00 --- /dev/null +++ b/callgrind/gostack.c @@ -0,0 +1,645 @@ +/*--------------------------------------------------------------------*/ +/*--- Callgrind ---*/ +/*--- gostack.c ---*/ +/*--------------------------------------------------------------------*/ + +/* + This file is part of Callgrind, a Valgrind tool for call tracing. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + The GNU General Public License is contained in the file COPYING. +*/ + +/* Go goroutine stacks. + * + * Callgrind keeps one shadow call stack per thread and pops frames by + * comparing stack pointers. The Go runtime runs many goroutines on one + * thread, each on its own heap-allocated stack, and moves between them + * without call/return pairs: + * + * - runtime.gogo jumps into another goroutine (or a new one). + * - runtime.mcall leaves a goroutine for the thread's g0 stack, where the + * scheduler picks the next goroutine. + * - runtime.systemstack and runtime.morestack run a function on g0 and + * then come back to the calling goroutine. + * + * Comparing SPs across these stacks pops unrelated frames, and frames of one + * goroutine end up under another one. So for Go programs each goroutine gets + * its own shadow call stack, keyed by its g pointer (held in a fixed register + * by the Go ABI). gogo and mcall swap shadow stacks. systemstack and + * morestack keep the goroutine's shadow stack, so work done on g0 for the + * goroutine stays under its caller, and tag the frames that run on g0 so the + * SP comparison never pops a goroutine frame from g0, or the other way round. + * + * Only the functions above are looked at. They are found by name, so this + * costs nothing for programs without a Go runtime. + * + * Only amd64 is supported for now. Other architectures keep the plain + * SP-based shadow stack. + */ + +#include "global.h" +#include "pub_tool_threadstate.h" +#include "pub_tool_guest.h" + +#if defined(VGA_amd64) +/* ABIInternal keeps the current g in R14 */ +# define GO_G_REG guest_R14 +/* Bytes a call pushes: the return address */ +# define GO_CALL_PUSH 8 +#endif + +/* Tag of frames that run on g0 for a goroutine (systemstack, morestack). + * Never a valid g pointer. */ +#define GO_TAG_SYSTEM ((UWord)1) + +/* Initial entries of a new goroutine's shadow stacks: goroutines are many and + * mostly shallow, the stacks grow on demand. */ +#define GO_STACK_INITIAL_ENTRIES 16 + +Bool CLG_(go_runtime) = False; +/* g whose shadow stack is the current call stack, 0 if not known yet */ +UWord CLG_(go_owner) = 0; +/* Tag of the stack the running frame is on */ +UWord CLG_(go_cur) = 0; +/* Function that jumps back to its own entry after morestack, see resume() */ +fn_node* CLG_(go_restart) = 0; + +/* A suspended goroutine's shadow stack. The first two fields are the + * VgHashNode header. */ +typedef struct _go_stack go_stack; +struct _go_stack { + go_stack* next; + UWord key; /* g pointer */ + + call_stack calls; + fn_stack fns; + Context* cxt; + BBCC* bbcc; + BBCC* nonskipped; + Int jmps_passed; + FullCost suspend_cost; /* cost counters when the goroutine was left */ + fn_node* root; /* function marked active again on resume */ +}; + +#if defined(GO_G_REG) +typedef struct { + const HChar* name; + UInt kind; +} go_switch_fn; + +/* Names with and without the ".abi0" suffix Go 1.17 added to assembly + * functions. "gogo" is the static target runtime.gogo.abi0 jumps to. */ +static const go_switch_fn go_switch_fns[] = { + { "runtime.gogo.abi0", GO_SWITCH_GOGO }, + { "runtime.gogo", GO_SWITCH_GOGO }, + { "gogo", GO_SWITCH_GOGO }, + { "runtime.mcall", GO_SWITCH_MCALL }, + { "runtime.systemstack.abi0", GO_SWITCH_NESTED }, + { "runtime.systemstack", GO_SWITCH_NESTED }, + { "runtime.morestack.abi0", GO_SWITCH_MORESTACK }, + { "runtime.morestack", GO_SWITCH_MORESTACK }, + /* vDSO calls run on g0 */ + { "runtime.nanotime1.abi0", GO_SWITCH_NESTED }, + { "runtime.nanotime1", GO_SWITCH_NESTED }, + { "runtime.walltime.abi0", GO_SWITCH_NESTED }, + { "runtime.walltime", GO_SWITCH_NESTED }, + { "runtime.walltime1.abi0", GO_SWITCH_NESTED }, + { "runtime.walltime1", GO_SWITCH_NESTED }, + { "runtime.vgetrandom1", GO_SWITCH_NESTED }, + /* cgo calls run on g0 */ + { "runtime.asmcgocall.abi0", GO_SWITCH_NESTED }, + { "runtime.asmcgocall", GO_SWITCH_NESTED }, +}; +#endif + +void CLG_(go_classify_fn)(fn_node* fn) +{ +#if defined(GO_G_REG) + UInt i; + + for (i = 0; i < sizeof(go_switch_fns) / sizeof(go_switch_fns[0]); i++) { + if (VG_(strcmp)(fn->name, go_switch_fns[i].name) != 0) continue; + + /* A plain "gogo" only counts once the Go runtime was seen; its only + * caller, runtime.gogo.abi0, always runs first. */ + if (!CLG_(go_runtime) && fn->name[0] == 'g') return; + + fn->go_switch = go_switch_fns[i].kind; + CLG_(go_runtime) = True; + return; + } +#endif +} + +static fn_node* callee_fn(call_entry* ce) +{ + return ce->jcc ? ce->jcc->to->cxt->fn[0] : 0; +} + +#if defined(GO_G_REG) + +static UWord read_g(void) +{ + UWord g; + VG_(get_shadow_regs_area)(CLG_(current_tid), (UChar*)&g, 0, + offsetof(VexGuestArchState, GO_G_REG), + sizeof(g)); + return g; +} + +/* Tag of the frame that runs after the top frame returns */ +static UWord tag_below_top(void) +{ + Int csp = CLG_(current_call_stack).sp; + UWord tag = csp > 0 ? CLG_(current_call_stack).entry[csp-1].go_tag : 0; + return tag ? tag : CLG_(go_owner); +} + +/* Add to the active counts of the functions on the current call stack, + * the inverse of what push/pop_call_stack do: a suspended goroutine's frames + * must not count as recursion of the running one. */ +static void adjust_active(Int dir) +{ + Int i; + + for (i = 0; i < CLG_(current_call_stack).sp; i++) { + call_entry* ce = &CLG_(current_call_stack).entry[i]; + fn_node* to_fn = callee_fn(ce); + UInt* pdepth; + + if (!to_fn) continue; + if (CLG_(clo).skip_direct_recursion && + ce->jcc->from->cxt->fn[0] == to_fn) continue; + + pdepth = CLG_(get_fn_entry)(to_fn->number); + if (dir < 0 && *pdepth == 0) continue; + *pdepth += dir; + } +} + +/* The function a call stack started in has no call entry, but callgrind marks + * it active (see the recursion level in setup_bbcc). Clear that mark when the + * stack is suspended; returns the function to mark again on resume. */ +static fn_node* uncount_root(void) +{ + Context* cxt = CLG_(current_call_stack).sp > 0 + ? CLG_(current_call_stack).entry[0].cxt + : CLG_(current_state).cxt; + fn_node* fn = cxt ? cxt->fn[0] : 0; + UInt* pdepth; + + if (!fn) return 0; + pdepth = CLG_(get_fn_entry)(fn->number); + if (*pdepth == 0) return 0; + (*pdepth)--; + return fn; +} + +/* Unused call stack arrays, kept in go_stack nodes linked by . Cost + * arrays hanging off call entries come from a pool and cannot be freed, so + * the arrays of finished goroutines are reused instead of freed. */ +static go_stack* spare_stacks = 0; + +/* Suspended goroutines by g pointer. Global: goroutines move between + * threads. */ +static VgHashTable* go_stacks = 0; + +/* Move the current call stack and context into a new go_stack. */ +static go_stack* stack_out(UWord key) +{ + go_stack* s = (go_stack*) CLG_MALLOC("cl.gostack.so.1", sizeof(go_stack)); + + adjust_active(-1); + s->root = uncount_root(); + + s->key = key; + CLG_(copy_current_call_stack)(&s->calls); + CLG_(copy_current_fn_stack)(&s->fns); + s->cxt = CLG_(current_state).cxt; + s->bbcc = CLG_(current_state).bbcc; + s->nonskipped = CLG_(current_state).nonskipped; + s->jmps_passed = CLG_(current_state).jmps_passed; + s->suspend_cost = (FullCost) CLG_MALLOC("cl.gostack.so.2", + CLG_(sets).full->size * sizeof(ULong)); + CLG_(copy_cost)(CLG_(sets).full, s->suspend_cost, CLG_(current_state).cost); + return s; +} + +/* Make 's call stack and context current and free . */ +static void stack_in(go_stack* s) +{ + Int i; + + CLG_(set_current_call_stack)(&s->calls); + CLG_(set_current_fn_stack)(&s->fns); + CLG_(current_state).cxt = s->cxt; + CLG_(current_state).bbcc = s->bbcc; + CLG_(current_state).nonskipped = s->nonskipped; + CLG_(current_state).jmps_passed = s->jmps_passed; + + /* The time the goroutine was suspended is not part of its frames' + * inclusive cost. Not CLG_(add_diff_cost): it moves to . */ + for (i = 0; i < CLG_(current_call_stack).sp; i++) { + call_entry* ce = &CLG_(current_call_stack).entry[i]; + Int e; + if (!ce->jcc || !ce->enter_cost) continue; + for (e = 0; e < CLG_(sets).full->size; e++) + ce->enter_cost[e] += CLG_(current_state).cost[e] - s->suspend_cost[e]; + } + + adjust_active(1); + if (s->root) (*CLG_(get_fn_entry)(s->root->number))++; + + VG_(free)(s->suspend_cost); + VG_(free)(s); +} + +/* Install an empty call stack for a goroutine run for the first time. */ +static void stack_fresh(void) +{ + go_stack* spare = spare_stacks; + call_stack cs; + fn_stack fs; + UInt i; + + if (spare) { + spare_stacks = spare->next; + cs = spare->calls; + fs = spare->fns; + VG_(free)(spare); + } + else { + cs.size = GO_STACK_INITIAL_ENTRIES; + cs.entry = (call_entry*) CLG_MALLOC("cl.gostack.sf.1", + cs.size * sizeof(call_entry)); + for (i = 0; i < cs.size; i++) cs.entry[i].enter_cost = 0; + + fs.size = GO_STACK_INITIAL_ENTRIES; + fs.bottom = (fn_node**) CLG_MALLOC("cl.gostack.sf.2", + fs.size * sizeof(fn_node*)); + } + cs.sp = 0; + cs.entry[0].cxt = 0; + fs.top = fs.bottom; + fs.bottom[0] = 0; + + CLG_(set_current_call_stack)(&cs); + CLG_(set_current_fn_stack)(&fs); + CLG_(current_state).cxt = 0; + CLG_(current_state).bbcc = 0; + CLG_(current_state).nonskipped = 0; + CLG_(current_state).jmps_passed = 0; +} + +/* Keep the arrays of for reuse and free the rest of it, without popping + * its frames: their goroutine is gone. */ +static void stack_free(go_stack* s) +{ + VG_(free)(s->suspend_cost); + s->suspend_cost = 0; + s->next = spare_stacks; + spare_stacks = s; +} + +/* Pop all frames of the current call stack and keep its arrays for reuse. + * A call stack must be installed again afterwards. */ +static void drop_current_stack(void) +{ + go_stack* s = (go_stack*) CLG_MALLOC("cl.gostack.dcs.1", sizeof(go_stack)); + + while (CLG_(current_call_stack).sp > 0) + CLG_(pop_call_stack)(); + uncount_root(); + + CLG_(copy_current_call_stack)(&s->calls); + CLG_(copy_current_fn_stack)(&s->fns); + s->suspend_cost = 0; + s->next = spare_stacks; + spare_stacks = s; +} + +/* A goroutine stops at a call into mcall or morestack and is resumed with + * SP where that call left it. The Go runtime moves goroutine stacks when + * they grow or shrink, so shift the shadow frames by the same distance. */ +static void relocate(Addr sp) +{ + Int csp = CLG_(current_call_stack).sp; + call_entry* top; + fn_node* fn; + Addr expected; + Int i; + + if (csp == 0) return; + top = &CLG_(current_call_stack).entry[csp-1]; + fn = callee_fn(top); + if (!fn || fn->go_switch == GO_SWITCH_NONE) return; + + expected = top->sp + GO_CALL_PUSH; + if (sp == expected) return; + + for (i = 0; i < csp; i++) + CLG_(current_call_stack).entry[i].sp += sp - expected; +} + +/* Continue the current goroutine at , where it stopped in mcall or + * morestack. A function that ran out of stack calls morestack from its + * prologue and, once the stack has grown, jumps back to its own entry. Mark + * that function so the jump does not count as a call of itself. */ +static void resume(Addr sp) +{ + static const HChar morestack_prefix[] = "runtime.morestack"; + fn_node* fn; + Int i; + + relocate(sp); + + i = CLG_(current_call_stack).sp - 1; + if (i < 0) return; + fn = callee_fn(&CLG_(current_call_stack).entry[i]); + if (!fn || fn->go_switch != GO_SWITCH_MORESTACK) return; + + /* Below morestack, and morestack_noctxt jumping to it */ + for (; i >= 0; i--) { + fn = callee_fn(&CLG_(current_call_stack).entry[i]); + if (!fn || VG_(strncmp)(fn->name, morestack_prefix, + sizeof(morestack_prefix) - 1) != 0) break; + } + CLG_(go_restart) = i >= 0 ? fn : 0; +} + +/* Pop the frames on top that run on g0 for the current goroutine. */ +static void pop_nested_frames(void) +{ + for (;;) { + Int csp = CLG_(current_call_stack).sp; + UWord tag; + if (csp == 0) return; + tag = CLG_(current_call_stack).entry[csp-1].go_tag; + if (tag == 0 || tag == CLG_(go_owner)) return; + CLG_(pop_call_stack)(); + } +} + +/* Suspend the current call stack under its goroutine. If the goroutine is + * not known (no switch function was entered yet), its frames are popped. + * + * The scheduler may run inside a nested g0 call (newstack preempting the + * goroutine), so pop those g0 frames: the goroutine resumes below them. */ +static void leave_current(void) +{ + if (!go_stacks) go_stacks = VG_(HT_construct)("cl.gostack.lc.1"); + + if (CLG_(go_owner)) { + pop_nested_frames(); + VG_(HT_add_node)(go_stacks, stack_out(CLG_(go_owner))); + return; + } + drop_current_stack(); +} + +/* A goroutine can be descheduled while it runs on g0 for itself, without + * mcall: newstack preempts it and the scheduler finds nothing else to run, + * so the thread sleeps with the goroutine's call stack still current. Take + * that call stack when another thread resumes the goroutine. */ +static go_stack* steal_from_other_thread(UWord g) +{ + thread_info** threads = CLG_(get_threads)(); + ThreadId me = CLG_(current_tid); + ThreadId tid; + go_stack* s; + + for (tid = 1; tid < VG_N_THREADS; tid++) { + thread_info* t = threads[tid]; + if (!t || tid == me || t->go_owner != g) continue; + + CLG_(switch_thread)(tid); + if (CLG_(current_state).sig > 0) { + /* Not reachable below a signal handler; forget the goroutine */ + CLG_(go_owner) = CLG_(go_cur) = 0; + CLG_(switch_thread)(me); + return 0; + } + pop_nested_frames(); + s = stack_out(g); + stack_fresh(); + CLG_(go_owner) = CLG_(go_cur) = 0; + CLG_(switch_thread)(me); + return s; + } + return 0; +} + +/* gogo jumps to the goroutine , starting it if is a function entry. */ +static Bool gogo_to(UWord g, BB* bb, Addr sp) +{ + go_stack* s; + + if (g == 0 || g == GO_TAG_SYSTEM) return False; + + if (g == CLG_(go_owner)) { + /* Back to the goroutine whose stack we are on, after it ran on g0 + * (morestack -> newstack -> gogo). */ + pop_nested_frames(); + resume(sp); + CLG_(go_cur) = tag_below_top(); + return False; + } + + leave_current(); + + s = VG_(HT_remove)(go_stacks, g); + if (!s) s = steal_from_other_thread(g); + if (s && bb->is_entry) { + /* g was reused for a new goroutine; the old one exited */ + stack_free(s); + s = 0; + } + + if (s) { + stack_in(s); + resume(sp); + } + else + stack_fresh(); + + CLG_(go_owner) = g; + CLG_(go_cur) = tag_below_top(); + return True; +} + +/* mcall runs a function on g0 that never returns; g0 starts over from the + * bottom of its stack. */ +static void mcall_to(UWord g0) +{ + go_stack* s; + + if (g0 == 0 || g0 == GO_TAG_SYSTEM || g0 == CLG_(go_owner)) return; + + leave_current(); + + /* Pop what g0 was doing when it last left (the scheduler up to gogo) */ + s = VG_(HT_remove)(go_stacks, g0); + if (s) { + stack_in(s); + drop_current_stack(); + } + stack_fresh(); + + CLG_(go_owner) = g0; + CLG_(go_cur) = g0; +} + +#endif /* GO_G_REG */ + +Bool CLG_(go_frame_blocks_unwind)(call_entry* ce, fn_node* running) +{ + fn_node* fn; + + /* Still inside the switch function: its SP is on either stack */ + fn = callee_fn(ce); + if (fn && fn == running && fn->go_switch != GO_SWITCH_NONE) return True; + + /* A frame of another stack */ + return ce->go_tag && CLG_(go_cur) && ce->go_tag != CLG_(go_cur); +} + +void CLG_(go_after_pop)(void) +{ +#if defined(GO_G_REG) + CLG_(go_cur) = tag_below_top(); +#endif +} + +/* Called for every BB of a Go program before the jump into is + * classified. Returns GO_SYNC_SWAPPED if the current call stack was swapped + * for another goroutine's, so the caller must reload its view of the previous + * BB, and GO_SYNC_RESTART if the jump into must not become a call. */ +Int CLG_(go_sync)(BB* bb, BB* last_bb, Addr sp) +{ +#if defined(GO_G_REG) + fn_node* fn = bb->fn; + fn_node* last_fn = last_bb ? last_bb->fn : 0; + fn_node* restart = CLG_(go_restart); + ClgJumpKind jmpkind; + + /* Go never switches goroutines inside a signal handler */ + if (CLG_(current_state).sig > 0) return GO_SYNC_NONE; + + if (restart) { + /* Still in the restarting function until it jumps to its entry */ + if (fn != restart || bb->is_entry) CLG_(go_restart) = 0; + if (fn == restart && bb->is_entry && last_fn == restart) + return GO_SYNC_RESTART; + } + + if (fn->go_switch != GO_SWITCH_NONE && fn != last_fn) { + /* Learn which goroutine we are on: the g register is valid when a + * switch function is entered */ + if (CLG_(go_owner) == 0) + CLG_(go_owner) = CLG_(go_cur) = read_g(); + return GO_SYNC_NONE; + } + + if (!last_fn || last_fn->go_switch == GO_SWITCH_NONE || fn == last_fn) + return GO_SYNC_NONE; + + jmpkind = last_bb->jmp[CLG_(current_state).jmps_passed].jmpkind; + if (jmpkind == jk_Return) return GO_SYNC_NONE; + + switch (last_fn->go_switch) { + case GO_SWITCH_GOGO: + return gogo_to(read_g(), bb, sp) ? GO_SYNC_SWAPPED : GO_SYNC_NONE; + case GO_SWITCH_MCALL: + mcall_to(read_g()); + return GO_SYNC_SWAPPED; + case GO_SWITCH_NESTED: + case GO_SWITCH_MORESTACK: + /* A call into the function to run on g0. Returning from it is handled + * by the pop. */ + CLG_(go_cur) = GO_TAG_SYSTEM; + return GO_SYNC_NONE; + default: + return GO_SYNC_NONE; + } +#else + return GO_SYNC_NONE; +#endif +} + +void CLG_(go_init_thread)(thread_info* t) +{ + t->go_owner = 0; + t->go_cur = 0; + t->go_restart = 0; +} + +void CLG_(go_save_thread)(thread_info* t) +{ + t->go_owner = CLG_(go_owner); + t->go_cur = CLG_(go_cur); + t->go_restart = CLG_(go_restart); +} + +void CLG_(go_load_thread)(thread_info* t) +{ + CLG_(go_owner) = t->go_owner; + CLG_(go_cur) = t->go_cur; + CLG_(go_restart) = t->go_restart; +} + +/* The current thread's call stack was popped: start over without a known + * goroutine. */ +void CLG_(go_unwind_thread)(void) +{ + CLG_(go_owner) = 0; + CLG_(go_cur) = 0; + CLG_(go_restart) = 0; +} + +/* Pop the frames of all suspended goroutines. */ +void CLG_(go_unwind_all)(void) +{ +#if defined(GO_G_REG) + call_stack cs; + fn_stack fs; + exec_state saved; + go_stack* s; + UInt i, n; + UWord* keys; + + if (!go_stacks || VG_(HT_count_nodes)(go_stacks) == 0) return; + + CLG_(copy_current_call_stack)(&cs); + CLG_(copy_current_fn_stack)(&fs); + saved = CLG_(current_state); + + n = VG_(HT_count_nodes)(go_stacks); + keys = (UWord*) CLG_MALLOC("cl.gostack.ua.1", n * sizeof(UWord)); + VG_(HT_ResetIter)(go_stacks); + for (i = 0; i < n; i++) keys[i] = ((go_stack*)VG_(HT_Next)(go_stacks))->key; + + for (i = 0; i < n; i++) { + s = VG_(HT_remove)(go_stacks, keys[i]); + stack_in(s); + drop_current_stack(); + } + VG_(free)(keys); + + CLG_(set_current_call_stack)(&cs); + CLG_(set_current_fn_stack)(&fs); + CLG_(current_state) = saved; +#endif +} diff --git a/callgrind/main.c b/callgrind/main.c index 69af08255..53fc3aa0d 100644 --- a/callgrind/main.c +++ b/callgrind/main.c @@ -1463,6 +1463,9 @@ void CLG_(unwind_thread)(thread_info* t) while(CLG_(current_call_stack).sp>0) CLG_(pop_call_stack)(); + /* no goroutine is known to run on the empty stack */ + CLG_(go_unwind_thread)(); + /* reset context and function stack for context generation */ CLG_(init_exec_state)( &CLG_(current_state) ); CLG_(current_fn_stack).top = CLG_(current_fn_stack).bottom; @@ -1492,6 +1495,7 @@ void CLG_(set_instrument_state)(const HChar* reason, Bool state) /* reset internal state: call stacks, simulator */ CLG_(forall_threads)(CLG_(unwind_thread)); + CLG_(go_unwind_all)(); CLG_(forall_threads)(zero_state_cost); (*CLG_(cachesim).clear)(); @@ -2016,6 +2020,7 @@ void finish(void) /* pop all remaining items from CallStack for correct sum */ CLG_(forall_threads)(CLG_(unwind_thread)); + CLG_(go_unwind_all)(); CLG_(dump_profile)(0, False); diff --git a/callgrind/threads.c b/callgrind/threads.c index 642813e4c..bfe5f5deb 100644 --- a/callgrind/threads.c +++ b/callgrind/threads.c @@ -238,6 +238,7 @@ thread_info* new_thread(ThreadId tid) CLG_(init_fn_array)( &(t->fn_active) ); CLG_(init_bbcc_hash)( &(t->bbccs) ); CLG_(init_jcc_hash)( &(t->jccs) ); + CLG_(go_init_thread)(t); return t; } @@ -255,6 +256,7 @@ static void load_thread_state(thread_info* t) CLG_(set_current_fn_stack) ( &(t->fns) ); CLG_(set_current_fn_array) ( &(t->fn_active) ); + CLG_(go_load_thread)(t); CLG_(current_thread_serial) = t->serial; /* If we cumulate costs of threads, use TID 1 for all jccs/bccs */ if (!CLG_(clo).separate_threads) t = thread[1]; @@ -282,6 +284,7 @@ void CLG_(switch_thread)(ThreadId tid) CLG_(copy_current_fn_stack) ( &(t->fns) ); CLG_(copy_current_fn_array) ( &(t->fn_active) ); + CLG_(go_save_thread)(t); /* If we cumulate costs of threads, use TID 1 for all jccs/bccs */ if (!CLG_(clo).separate_threads) t = thread[1]; CLG_(copy_current_bbcc_hash)( &(t->bbccs) ); @@ -336,6 +339,7 @@ void CLG_(pre_signal)(ThreadId tid, Int sigNum, Bool alt_stack) CLG_(zero_cost)( CLG_(sets).full, es->cost ); CLG_(current_state).cost = es->cost; es->call_stack_bottom = CLG_(current_call_stack).sp; + es->on_altstack = alt_stack; /* setup current state for a spontaneous call */ CLG_(init_exec_state)( &CLG_(current_state) ); @@ -358,6 +362,34 @@ void CLG_(run_post_signal_on_call_stack_bottom)(void) CLG_(post_signal)( CLG_(current_tid), CLG_(current_state).sig ); } +/* True if the call stack is at the bottom of a signal handler running on + * the thread's sigaltstack, and `sp` still lies inside that altstack. + * + * The frames below the handler's call_stack_bottom live on the interrupted + * stack, so their SPs cannot be compared with an SP on the altstack: the + * altstack may be mapped above the interrupted stack (e.g. Go's per-M signal + * stacks vs. its heap-allocated goroutine stacks). Without this check the + * SP-based unwind pops through the bottom, which looks like a longjmp out of + * the handler, and the handler's real rt_sigreturn then fails the signal + * number check in post_signal. + */ +Bool CLG_(sp_in_altstack_signal_handler)(Addr sp) +{ + exec_state* es; + Addr alt_min; + SizeT alt_size; + + if (CLG_(current_state).sig <= 0) return False; + + es = top_exec_state(); + if (!es->on_altstack) return False; + if (CLG_(current_call_stack).sp != es->call_stack_bottom) return False; + + alt_min = VG_(thread_get_altstack_min)(CLG_(current_tid)); + alt_size = VG_(thread_get_altstack_size)(CLG_(current_tid)); + return sp >= alt_min && sp - alt_min < alt_size; +} + void CLG_(post_signal)(ThreadId tid, Int sigNum) { exec_state* es;