test-companion: say why signing in as the GitHub App failed

Any failure to look up the App's installation was reported as "the GitHub
App is not installed on this repository", hiding GitHub's real answer.
A wrong App ID or a key from another App looked exactly like a missing
installation.

Now only a 404 is reported as "not installed". Any other failure is a
warning that carries GitHub's own reason, e.g. "Bad credentials
(HTTP 401)", and says which inputs to check. A failed token request
also carries GitHub's reason.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: 2 people

test-companion/src/app-auth.mjs

@@ -23,6 +23,32 @@ export function jwtClaims(appId, nowSeconds) {
 	return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) }
 }
 
+/**
+ * GitHub's reason a `gh api` call failed, e.g. "Bad credentials (HTTP 401)", for the log.
+ * gh's error output holds only the response message, never the token it sent.
+ */
+export function ghFailureReason(result) {
+	const line = String(result?.stderr ?? "")
+		.split("\n")
+		.map((l) => l.trim())
+		.find(Boolean)
+	return line ? line.replace(/^gh:\s*/, "") : "no detail from GitHub"
+}
+
+/** What to log when looking up the App's installation on this repository fails. */
+export function installationLookupMessage(result) {
+	const reason = ghFailureReason(result)
+	if (/\bHTTP 404\b/.test(reason)) {
+		return { level: "notice", text: "Test Companion: the GitHub App is not installed on this repository; using the workflow token." }
+	}
+	return {
+		level: "warning",
+		text:
+			`Test Companion: could not sign in as the GitHub App (${reason}); using the workflow token. ` +
+			"Check that app-id is the App's numeric ID and app-private-key is a key generated for that App.",
+	}
+}
+
 export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) {
 	const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" }))
 	const payload = b64url(JSON.stringify(jwtClaims(appId, now)))

test-companion/src/main.mjs

@@ -14,7 +14,13 @@ import { canIsolate, prepareSandbox, stopSandbox, handToAgent, writeAgentEnv, re
 import { agentEnvFrom, asAgentCommand, checkPrHandoff, readClaimResult } from "./sandbox-policy.mjs"
 import { startRelay } from "./relay.mjs"
 import { runCapturingAnswer } from "./answer.mjs"
-import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs"
+import {
+	appCredentials,
+	signJwt,
+	installationTokenRequest,
+	ghFailureReason,
+	installationLookupMessage,
+} from "./app-auth.mjs"
 import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs"
 import {
 	renderJobSummary,
@@ -181,7 +187,8 @@ function appInstallationToken() {
 
 	const found = asApp(["api", `repos/${REPO}/installation`])
 	if (found.status !== 0) {
-		notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.")
+		const { level, text } = installationLookupMessage(found)
+		;(level === "warning" ? warn : notice)(text)
 		return undefined
 	}
 	let installationId
@@ -213,7 +220,7 @@ function appInstallationToken() {
 	const minted = mint(scope)
 	if (minted.status !== 0) {
 		warn(
-			"Could not get a GitHub App token with the permissions Test Companion needs; using the workflow token instead. " +
+			`Could not get a GitHub App token with the permissions Test Companion needs (${ghFailureReason(minted)}); using the workflow token instead. ` +
 				"If the App's permissions were changed, an organisation owner may need to accept them.",
 		)
 		return undefined

test-companion/test/app-auth.test.mjs

@@ -1,7 +1,14 @@
 import { test, describe } from "node:test"
 import assert from "node:assert/strict"
 import { generateKeyPairSync, createVerify } from "node:crypto"
-import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from "../src/app-auth.mjs"
+import {
+	appCredentials,
+	jwtClaims,
+	signJwt,
+	installationTokenRequest,
+	ghFailureReason,
+	installationLookupMessage,
+} from "../src/app-auth.mjs"
 
 const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 })
 const pem = privateKey.export({ type: "pkcs1", format: "pem" })
@@ -88,3 +95,27 @@ describe("installationTokenRequest", () => {
 		assert.equal(installationTokenRequest(undefined), undefined)
 	})
 })
+
+describe("App sign-in failures say why", () => {
+	test("GitHub's reason is taken from gh's error output", () => {
+		assert.equal(ghFailureReason({ stderr: "gh: Bad credentials (HTTP 401)\n" }), "Bad credentials (HTTP 401)")
+	})
+
+	test("an empty error output still gives a reason", () => {
+		assert.equal(ghFailureReason({ stderr: "" }), "no detail from GitHub")
+		assert.equal(ghFailureReason(undefined), "no detail from GitHub")
+	})
+
+	test("only a 404 means the App is not installed", () => {
+		const m = installationLookupMessage({ stderr: "gh: Not Found (HTTP 404)" })
+		assert.equal(m.level, "notice")
+		assert.match(m.text, /not installed on this repository/)
+	})
+
+	test("any other failure is a warning that carries GitHub's reason", () => {
+		const m = installationLookupMessage({ stderr: "gh: A JSON web token could not be decoded (HTTP 401)" })
+		assert.equal(m.level, "warning")
+		assert.match(m.text, /A JSON web token could not be decoded \(HTTP 401\)/)
+		assert.doesNotMatch(m.text, /not installed/)
+	})
+})