feat(diagnostics): say what the app was doing when it froze

konture-test/src/test/kotlin/com/gitvantage/ArchitectureTest.kt

@@ -235,7 +235,7 @@ val Architecture by testSuite {
     }
 
     /**
-     * Starting a process is allowed in four places, and only one of them is about git:
+     * Starting a process is allowed in five places, and only one of them is about git:
      *
      *  - `git/Git` runs the git binary — the subject of this file;
      *  - `Actions` launches what the user picked from "Open in …" (a terminal, an IDE, `xdg-open`,
@@ -245,6 +245,8 @@ val Architecture by testSuite {
      *    `gsettings` and `kreadconfig`, because Skiko cannot: it dlopens an unversioned
      *    `libdbus-1.so` that only a machine with the -devel package installed has, and reports
      *    UNKNOWN otherwise (SKIKO-1177). Shells out only on that UNKNOWN, and only on Linux.
+     *  - `app/Diagnostics` asks `ps` for the process's resident memory on macOS, which has no
+     *    `/proc` and no Java API for it. Once a minute, and only while performance collection is on.
      *
      * Each entry has a reason, which is what makes a new name appearing in a failure worth stopping
      * on rather than adding to the list.
@@ -262,10 +264,10 @@ val Architecture by testSuite {
      * file: Konture attributes a dependency to the file it appears in, so an allowlist over classes
      * would have to name every class each allowed file declares.
      */
-    test("process spawning is confined to Git and the three non-git callers") {
+    test("process spawning is confined to Git and the four non-git callers") {
         architecture {
             files {
-                that().notNamed { it in setOf("Git.kt", "Actions.kt", "GitHub.kt", "Theme.kt") }
+                that().notNamed { it in setOf("Git.kt", "Actions.kt", "GitHub.kt", "Theme.kt", "Diagnostics.kt") }
                 should().notReferenceClass("java.lang.ProcessBuilder")
             }
         }

src/main/kotlin/com/gitvantage/app/AllocationMeter.kt

@@ -0,0 +1,137 @@
+// SPDX-FileCopyrightText: 2026 Raman Gupta
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package com.gitvantage.app
+
+import java.lang.management.ManagementFactory
+
+/**
+ * How fast the app allocates, and which threads do it.
+ *
+ * The collector's figures say how often it ran and for how long; they cannot say how much was
+ * allocated to make it run, and that is the number a busy collector is a symptom of. A field report
+ * showed 2.5 young collections a second and 6% of the time in GC with a steady ~55MB live — about
+ * 1GB/s allocated, if the young generation was the 10% of max heap the native image defaults to —
+ * and that had to be worked out by hand from a flag default, because nothing measured it.
+ *
+ * Grouped by thread because the groups are where the possible answers live: the UI thread is
+ * composition and text layout, the dispatcher workers are scans and the `git` output they read, the
+ * watcher threads are filesystem events. A total alone would confirm the problem and leave the
+ * cause exactly where it was.
+ *
+ * Measured from the per-thread allocation counters, which a native image may not support; when it
+ * doesn't, the line says so rather than reporting zero. A thread that ends between two samples takes
+ * whatever it allocated since the last one with it, so the figures are a slight undercount.
+ */
+object AllocationMeter {
+
+    /** Where a thread's allocations are filed — see [groupOf]. */
+    enum class Group(val label: String) { UI("ui"), WORKERS("workers"), WATCHER("watcher"), OTHER("other") }
+
+    private val bean: com.sun.management.ThreadMXBean? =
+        runCatching { ManagementFactory.getThreadMXBean() as? com.sun.management.ThreadMXBean }.getOrNull()
+            ?.takeIf { runCatching { it.isThreadAllocatedMemorySupported && it.isThreadAllocatedMemoryEnabled }.getOrDefault(false) }
+
+    /** Each live thread's allocation counter when it was last read. */
+    private val lastSeen = HashMap<Long, Long>()
+
+    /** Everything attributed since the process started, by group. */
+    private val total = LongArray(Group.entries.size)
+
+    /** Attributed since the last [interval], by group. */
+    private val recent = LongArray(Group.entries.size)
+    private var recentSince = System.nanoTime()
+
+    /** Whether [start] has taken the baseline [recent] is measured from. */
+    private var baselined = false
+
+    /**
+     * Which group a thread belongs to, by its name — and by identity for the UI thread, whose name
+     * is the platform's ("main" or an AppKit thread) and says nothing.
+     */
+    internal fun groupOf(name: String, isUi: Boolean): Group = when {
+        isUi -> Group.UI
+        name.startsWith("DefaultDispatcher-worker") -> Group.WORKERS
+        name.startsWith("notify-rs") -> Group.WATCHER
+        else -> Group.OTHER
+    }
+
+    /** Read every live thread's counter and file what it allocated since the last read. */
+    private fun sample(bean: com.sun.management.ThreadMXBean) {
+        val ids = bean.allThreadIds
+        // Depth 0: names and nothing else. Stacks would cost a safepoint per sample for no reason.
+        val infos = bean.getThreadInfo(ids, 0)
+        val bytes = bean.getThreadAllocatedBytes(ids)
+        val ui = StallWatchdog.uiThreadId
+        val live = HashSet<Long>(ids.size)
+        ids.forEachIndexed { i, id ->
+            val now = bytes[i]
+            val name = infos[i]?.threadName
+            if (now < 0 || name == null) return@forEachIndexed // ended since it was listed
+            live += id
+            val delta = now - (lastSeen.put(id, now) ?: 0L)
+            if (delta <= 0) return@forEachIndexed
+            val g = groupOf(name, id == ui).ordinal
+            total[g] += delta
+            recent[g] += delta
+        }
+        lastSeen.keys.retainAll(live)
+    }
+
+    /**
+     * The allocation rate since the last call, for the periodic perf dump: "alloc 980MB/s over 60s
+     * (workers 71% · ui 22% · …)", or a line saying the counters aren't available.
+     */
+    @Synchronized
+    fun interval(): String {
+        val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)"
+        if (!baselined) {
+            start()
+            return "alloc measuring from now"
+        }
+        runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" }
+        val now = System.nanoTime()
+        val seconds = ((now - recentSince) / 1_000_000_000.0).coerceAtLeast(0.001)
+        val line = "alloc %.1fMB/s over %ds".format(recent.sum() / seconds / (1024 * 1024), seconds.toLong()) +
+            breakdown(recent)
+        recent.fill(0)
+        recentSince = now
+        return line
+    }
+
+    /**
+     * Take the baseline the next [interval] is measured from — whenever collection switches on, so
+     * the first reading covers the time it has been on rather than the whole session.
+     *
+     * Without one the first reading would file everything each thread allocated since it started
+     * under an interval of no length at all; [interval] takes it itself if nobody did.
+     */
+    @Synchronized
+    fun start() {
+        val b = bean ?: return
+        runCatching { sample(b) }
+        recent.fill(0)
+        recentSince = System.nanoTime()
+        baselined = true
+    }
+
+    /** Everything attributed since the process started, for the clipboard report. Doesn't move [interval]'s baseline. */
+    @Synchronized
+    fun total(): String {
+        val b = bean ?: return "alloc unavailable (this runtime has no per-thread allocation counters)"
+        runCatching { sample(b) }.onFailure { return "alloc unavailable (${it.javaClass.simpleName})" }
+        return "allocated ${mb(total.sum().toDouble())}MB since start" + breakdown(total)
+    }
+
+    private fun breakdown(byGroup: LongArray): String {
+        val sum = byGroup.sum()
+        if (sum <= 0) return ""
+        return Group.entries
+            .map { it to byGroup[it.ordinal] }
+            .filter { it.second > 0 }
+            .sortedByDescending { it.second }
+            .joinToString(" · ", prefix = " (", postfix = ")") { (g, n) -> "${g.label} ${n * 100 / sum}%" }
+    }
+
+    private fun mb(bytes: Double): Long = (bytes / (1024 * 1024)).toLong()
+}

src/main/kotlin/com/gitvantage/app/AppState.kt

@@ -36,6 +36,7 @@ import com.gitvantage.model.Repo
 import com.gitvantage.model.RepoCandidate
 import com.gitvantage.model.WatchAction
 import com.gitvantage.model.WatchPolicy
+import com.gitvantage.model.WatchStats
 import com.gitvantage.model.Worktree
 import com.gitvantage.model.WorktreeAlert
 import com.gitvantage.model.WorktreeAlerts
@@ -558,6 +559,7 @@ class AppState(private val scope: CoroutineScope) {
             runCatching {
                 w.events.collect { ev ->
                     Perf.count("watch.events")
+                    if (Perf.enabled) recordWatchEvent(ev)
                     when (val action = actionFor(ev)) {
                         is WatchAction.Rescan -> noteWatchEvent(action.id)
                         WatchAction.RescanAll -> { Perf.count("watch.overflow"); scheduleFullRescan() }
@@ -646,6 +648,7 @@ class AppState(private val scope: CoroutineScope) {
                 want.forEach { id -> if (id !in watchRegs) register(w, id) }
                 (watchRegs.keys - want).forEach { id ->
                     runCatching { watchRegs.remove(id)?.forEach { it.close() } }
+                    watchRootById.remove(id)
                     watchBursts.remove(id)
                 }
             }
@@ -656,11 +659,38 @@ class AppState(private val scope: CoroutineScope) {
     private fun register(w: FsWatcher, id: String) {
         // A list of one, today. Every registration carries the repo id as its name, which is what
         // event routing reads, so the shape survives if [WatchRoots] ever returns several again.
+        watchRootById[id] = watchRootFor(id)
         watchRegs[id] = WatchRoots.forRepo(id).mapNotNull { root ->
             runCatching { w.watch(watchRootFor(root.path), root.recursive, id) }.getOrNull()
         }
     }
 
+    /**
+     * Each watched repo's root as the watcher was handed it, which is the spelling event paths come
+     * back under — what [recordWatchEvent] needs to say which folder an event was in. Resolved once
+     * at registration rather than per event, since resolving touches the filesystem.
+     */
+    private val watchRootById = ConcurrentHashMap<String, Path>()
+
+    /** File [ev] under its repo and folder for [WatchStats]. Only while collecting; runs per event. */
+    private fun recordWatchEvent(ev: FsWatchEvent) {
+        val id = ev.source?.name ?: return
+        val root = watchRootById[id] ?: return
+        val folder = eventPaths(ev).firstOrNull()?.let { WatchStats.folderOf(root, it) } ?: "(overflow)"
+        WatchStats.event(id, folder)
+    }
+
+    /**
+     * How a repo appears in a diagnostics report: its position in the list and how many worktrees it
+     * has, never its path or name — the report is meant to be pasted into a public issue. The
+     * position is enough for the person who sent it to say which repo it is, if asked.
+     */
+    private fun diagLabel(id: String): String {
+        val n = order.indexOf(id) + 1
+        val trees = repos.firstOrNull { it.id == id }?.worktrees?.size ?: 0
+        return if (n > 0) "repo #$n ($trees ${if (trees == 1) "worktree" else "worktrees"})" else "(untracked repo)"
+    }
+
     /**
      * What [ev] asks of the app — see [WatchPolicy.actionFor].
      *
@@ -740,6 +770,7 @@ class AppState(private val scope: CoroutineScope) {
                         else due.filter { entries[it] != null }
                     if (ids.isNotEmpty()) {
                         Perf.count("watch.rescans", ids.size.toLong())
+                        ids.forEach(WatchStats::rescan)
                         Perf.timed("watch.rescan.batch") { rescanRepos(ids, fetch = false) }
                     }
                 }
@@ -1400,6 +1431,8 @@ class AppState(private val scope: CoroutineScope) {
         // Started whether or not collection is on, since it can be switched on at any point; an
         // idle timer once a minute is cheaper than starting and stopping one correctly.
         watchScope.launch {
+            // Only when collecting: switching on later takes its own baseline (see setPerfCollection).
+            if (Perf.enabled) AllocationMeter.start()
             while (currentCoroutineContext().isActive) {
                 delay(PERF_REPORT_MS.milliseconds)
                 if (Perf.enabled) dumpPerf("every ${PERF_REPORT_MS / 1000}s")
@@ -1411,8 +1444,10 @@ class AppState(private val scope: CoroutineScope) {
     private fun dumpPerf(label: String) {
         val lines = Perf.report(markInterval = true)
         if (lines.isEmpty()) return
+        val vitals = Diagnostics.vitals() + AllocationMeter.interval()
         DiagLog.write(
-            "GitVantage perf [$label] · ${Diagnostics.vitals().joinToString(" · ")}\n" + lines.joinToString("\n"),
+            "GitVantage perf [$label] · ${vitals.joinToString(" · ")}\n" +
+                (lines + WatchStats.report(label = ::diagLabel)).joinToString("\n"),
         )
     }
 
@@ -1427,7 +1462,12 @@ class AppState(private val scope: CoroutineScope) {
      */
     fun setPerfCollection(on: Boolean) {
         if (on == perfCollecting) return
+        // `GITVANTAGE_PERF=1` can't be undone from here, which is why Settings dims Off. Ignore the
+        // click outright: going on would log "collection switched off" for collection that carries
+        // on, and save "off" as a choice the user can't see took effect.
+        if (!on && Perf.forcedOn) return
         if (!on) watchScope.launch { dumpPerf("collection switched off") }
+        if (on) watchScope.launch { AllocationMeter.start() }
         Perf.enable(on)
         perfCollecting = Perf.enabled
         Registry.saveSettings(Registry.settings().copy(collectPerf = on))
@@ -1442,7 +1482,8 @@ class AppState(private val scope: CoroutineScope) {
     suspend fun diagnosticsReport(): String {
         val inventory = "${repos.size} repos · ${repos.sumOf { it.worktrees.size }} worktrees · " +
             "GitHub polling ${if (githubEnabled) "on" else "off"}"
-        return withContext(Dispatchers.IO) { Diagnostics.report(inventory) }
+        val watched = WatchStats.report(label = ::diagLabel)
+        return withContext(Dispatchers.IO) { Diagnostics.report(inventory, watched) }
     }
 
     /** Show the folder holding the log, for attaching the file itself rather than an excerpt. */

src/main/kotlin/com/gitvantage/app/Diagnostics.kt

@@ -5,7 +5,9 @@ package com.gitvantage.app
 
 import com.gitvantage.BuildInfo
 import com.gitvantage.model.Perf
+import java.io.File
 import java.lang.management.ManagementFactory
+import java.util.concurrent.TimeUnit
 
 /**
  * The report someone pastes into an issue, and the pieces it is assembled from.
@@ -59,6 +61,7 @@ object Diagnostics {
             "heap used=${mb(rt.totalMemory() - rt.freeMemory())}MB committed=${mb(rt.totalMemory())}MB " +
                 "max=${mb(rt.maxMemory())}MB",
         )
+        residentMb()?.let { add("rss ${it}MB") }
         runCatching {
             ManagementFactory.getGarbageCollectorMXBeans()
                 .filter { it.collectionCount > 0 }
@@ -68,22 +71,58 @@ object Diagnostics {
     }
 
     /**
-     * The whole report: [environment], [inventory], [vitals], the counters, then the recent log.
+     * The process's resident memory in MB, or null where it can't be read.
+     *
+     * The heap figures are only the Java part. A field report's 1.7GB footprint sat beside ~55MB of
+     * live heap, and nothing logged could say how the rest grew — the collector keeping memory it had
+     * touched, or native memory (graphics, the watcher library) the heap never sees. This is the
+     * number that answers it over a long session.
+     *
+     * Linux reads `/proc`. macOS has no such file and no Java API for it, so it asks `ps` — a process
+     * per call, which is why this is read once a minute while collecting and not more. Windows reports
+     * nothing.
+     */
+    internal fun residentMb(): Long? = runCatching {
+        val os = System.getProperty("os.name").orEmpty()
+        when {
+            os.startsWith("Linux") ->
+                File("/proc/self/status").useLines { lines ->
+                    lines.firstOrNull { it.startsWith("VmRSS:") }?.let(::parseVmRssKb)
+                }?.let { it / 1024 }
+            os.startsWith("Mac") -> {
+                val p = ProcessBuilder("ps", "-o", "rss=", "-p", ProcessHandle.current().pid().toString())
+                    .redirectError(ProcessBuilder.Redirect.DISCARD).start()
+                val out = p.inputStream.bufferedReader().readText()
+                if (p.waitFor(5, TimeUnit.SECONDS)) out.trim().toLongOrNull()?.let { it / 1024 } else null
+            }
+            else -> null
+        }
+    }.getOrNull()
+
+    /** The kB figure from a `/proc` `VmRSS:` line — "VmRSS:   123456 kB". */
+    internal fun parseVmRssKb(line: String): Long? =
+        line.removePrefix("VmRSS:").trim().substringBefore(' ').toLongOrNull()
+
+    /**
+     * The whole report: [environment], [inventory], [vitals], the counters and [watched] — the
+     * caller's `WatchStats` lines, labelled by it — then the recent log.
      *
      * [inventory] is the caller's, because how much is being tracked is application state and this
      * object holds none. Reads the log from disk, so not for the UI thread.
      */
-    fun report(inventory: String): String = buildString {
+    fun report(inventory: String, watched: List<String> = emptyList()): String = buildString {
         appendLine(environment)
         appendLine(inventory)
         vitals().forEach { appendLine(it) }
+        appendLine(AllocationMeter.total())
         appendLine()
         val counters = Perf.report()
         when {
             counters.isNotEmpty() -> {
                 val since = Perf.collectingSeconds()?.let { "collecting for ${formatUptime(it)}" }
                 appendLine("Counters (${since ?: "collection now off"})")
                 counters.forEach { appendLine(it) }
+                watched.forEach { appendLine(it) }
             }
             Perf.enabled -> appendLine("Counters: collecting, nothing recorded yet")
             else -> appendLine("Counters: not collected — turn on \"Collect performance data\" in Settings")