Reject Copilot discovery-only responses as successful completion

actions/setup/js/copilot_harness.cjs

@@ -49,7 +49,7 @@
 
 "use strict";
 
-const { iterateJsonl } = require("./jsonl_helpers.cjs");
+const { iterateJsonl, readJsonlFile } = require("./jsonl_helpers.cjs");
 require("./shim.cjs");
 
 const { getErrorMessage } = require("./error_helpers.cjs");
@@ -105,7 +105,7 @@ const COPILOT_ROUTING_POLICY = getAWFModelRoutingPolicy("copilot");
 const { loadModelsJson } = require("./model_costs.cjs");
 const { applyCopilotWireAPI: configureCopilotWireAPI, warnCopilotSubagentEndpoints, recordModelEndpointMismatch, getSupportedEndpoints, MODEL_ENDPOINT_FIX } = require("./copilot_wire_api.cjs");
 const { loadModelAliasMap, resolveEngineModel, normalizeForCopilotCLI, ModelAliasResolutionError, recordModelResolution, splitModelIdentifier } = require("./resolve_model_alias.cjs");
-const { COPILOT_MODEL_VERIFICATION_EXIT_CODE, createCopilotModelObserver, readCopilotModelSubstitution, recordCopilotModelSelection } = require("./copilot_model_selection.cjs");
+const { COPILOT_MODEL_VERIFICATION_EXIT_CODE, copilotSessionStateDirectories, createCopilotModelObserver, readCopilotModelSubstitution, recordCopilotModelSelection } = require("./copilot_model_selection.cjs");
 const { getEngineSubAgentTarget } = require("./extract_inline_sub_agents.cjs");
 const { inlineArtifactTarget } = require("./inline_artifact_target.cjs");
 const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported|No model available\b[^\n]*policy enablement/i;
@@ -1216,14 +1216,59 @@ function extractTokenCountFromOutput(output) {
 
 /** @param {string} output @returns {boolean} */
 function isCopilotContentFilterError(output) {
-  for (const event of iterateJsonl(output)) {
-    if (event?.type !== "assistant.message") continue;
+  for (const event of iterateJsonl(output, { valueFilter: event => event?.type === "assistant.message" })) {
     const context = copilotSessionContext(event);
     if (context.agentId === undefined && !context.parentToolUseId && getCopilotMessageRefusal(event.data)?.reason === "content_filter") return true;
   }
   return false;
 }
 
+/** @param {Iterable<any>} events @returns {boolean} */
+function isCopilotToolSearchOnlyCompletion(events) {
+  let message;
+  for (const event of events) {
+    const context = copilotSessionContext(event);
+    if (context.agentId !== undefined || context.parentToolUseId) continue;
+    if (event?.type === "assistant.message") message = event.data;
+    if (event?.type === "assistant.refusal") message = undefined;
+  }
+  return (
+    !!message &&
+    (message.content === "" || message.content === null) &&
+    Array.isArray(message.toolRequests) &&
+    message.toolRequests.length === 0 &&
+    !getCopilotMessageRefusal(message) &&
+    message.serverTools?.provider === "openai-responses" &&
+    Array.isArray(message.serverTools.items) &&
+    message.serverTools.items.length > 0 &&
+    message.serverTools.items.every(item => item?.type === "tool_search_call" && item.execution === "server" && item.status === "completed")
+  );
+}
+
+const copilotAssistantResponseLineFilter = line => /"type"\s*:\s*"assistant\.(?:message|refusal)"/.test(line);
+const copilotAssistantResponseFilter = event => event?.type === "assistant.message" || event?.type === "assistant.refusal";
+
+/**
+ * Native CLI stdout is plain text; inspect only this invocation's exact session.
+ * Never select another session by recency in a shared HOME.
+ * @param {string} output
+ * @param {string | undefined} sessionId
+ * @returns {boolean}
+ */
+function hasCopilotToolSearchOnlyCompletion(output, sessionId) {
+  if (sessionId && /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i.test(sessionId)) {
+    for (const directory of copilotSessionStateDirectories()) {
+      try {
+        return isCopilotToolSearchOnlyCompletion(readJsonlFile(path.join(directory, sessionId, "events.jsonl"), { strict: true, valueFilter: copilotAssistantResponseFilter }));
+      } catch (error) {
+        const missing = error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT";
+        if (!missing) log("Could not inspect Copilot completion events; leaving missing-output validation to the collector");
+      }
+    }
+  }
+  return isCopilotToolSearchOnlyCompletion(iterateJsonl(output, { lineFilter: copilotAssistantResponseLineFilter, valueFilter: copilotAssistantResponseFilter }));
+}
+
 /**
  * Classify a failed Copilot attempt into a short, named failure class.
  * @param {{
@@ -2108,6 +2153,13 @@ async function main() {
             result.runtimeGuardReason = "copilot_model_verification";
             if (modelSubstitution) result.output += `\n${modelSubstitution}`;
           }
+          if (result.exitCode === 0 && (!safeOutputsPath || !hasTerminalSafeOutput(safeOutputsPath)) && hasCopilotToolSearchOnlyCompletion(result.output, sessionId)) {
+            result.exitCode = 1;
+            result.runtimeGuardReason = "copilot_tool_search_only_completion";
+            log(
+              "Copilot stopped after server-side tool discovery without text, a local tool request, or a terminal safe output. Continuation belongs to Copilot CLI/upstream Responses handling; task completion is unverified. Not retrying to avoid duplicate side effects."
+            );
+          }
           lastHasOutput = result.hasOutput;
           const attemptDetections = detectCopilotErrors(result.output);
           detectedCopilotErrors.inferenceAccessError ||= attemptDetections.inferenceAccessError;
@@ -2118,6 +2170,10 @@ async function main() {
           return result;
         },
         handleFailure: async ({ attempt, result }) => {
+          if (result.runtimeGuardReason === "copilot_tool_search_only_completion") {
+            result.failureClasses = ["copilot_tool_search_only_completion"];
+            return { action: "stop" };
+          }
           if (result.runtimeGuardReason === "copilot_model_verification") {
             result.failureClasses = ["copilot_model_verification"];
             return { action: "stop" };
@@ -2566,6 +2622,8 @@ if (typeof module !== "undefined" && module.exports) {
     detectCopilotErrors,
     classifyCopilotFailure,
     isCopilotContentFilterError,
+    isCopilotToolSearchOnlyCompletion,
+    hasCopilotToolSearchOnlyCompletion,
     extractTokenCountFromOutput,
     shouldRetryFailedExecution,
     shouldContinueCopilotSessionAfterFailure,

actions/setup/js/copilot_harness.test.cjs

@@ -15,6 +15,8 @@ const {
   buildMissingToolPermissionIssuePayload,
   classifyCopilotFailure,
   isCopilotContentFilterError,
+  isCopilotToolSearchOnlyCompletion,
+  hasCopilotToolSearchOnlyCompletion,
   extractTokenCountFromOutput,
   buildMissingToolAlternatives,
   buildInfrastructureIncompletePayload,
@@ -117,6 +119,115 @@ describe("native invocation model and session selection", () => {
   });
 });
 
+describe("Copilot tool-search-only completion", () => {
+  // Shape observed in dotnet/android run 40700757014; no opaque IDs or prompt payloads.
+  const search = {
+    type: "assistant.message",
+    data: {
+      content: "",
+      toolRequests: [],
+      serverTools: { provider: "openai-responses", items: [{ type: "tool_search_call", execution: "server", status: "completed" }] },
+    },
+  };
+  const tempDirs = [];
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    for (const dir of tempDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  it.each(["", null])("recognizes an empty final main response (%j)", content => {
+    expect(isCopilotToolSearchOnlyCompletion([{ ...search, data: { ...search.data, content } }, { type: "session.shutdown" }])).toBe(true);
+  });
+
+  it.each([
+    { content: "Review complete." },
+    { toolRequests: [{ name: "github.pull_request_read" }] },
+    { refusal: "Cannot comply." },
+    { serverTools: { provider: "openai-responses" } },
+    { serverTools: { provider: "openai-responses", items: [{ type: "tool_search_call", execution: "server", status: "in_progress" }] } },
+    { serverTools: { provider: "openai-responses", items: [{ type: "web_search_call", execution: "server", status: "completed" }] } },
+  ])("does not mistake another response for discovery-only completion (%j)", overrides => {
+    expect(isCopilotToolSearchOnlyCompletion([{ ...search, data: { ...search.data, ...overrides } }])).toBe(false);
+  });
+
+  it("uses the last main assistant message and ignores subagents", () => {
+    const done = { type: "assistant.message", data: { content: "Done." } };
+    expect(isCopilotToolSearchOnlyCompletion([search, done])).toBe(false);
+    expect(isCopilotToolSearchOnlyCompletion([done, { ...search, agentId: "child" }])).toBe(false);
+    expect(isCopilotToolSearchOnlyCompletion([done, { ...search, data: { ...search.data, parentToolCallId: "child-call" } }])).toBe(false);
+    expect(isCopilotToolSearchOnlyCompletion([search, { ...done, agentId: "child" }])).toBe(true);
+    expect(isCopilotToolSearchOnlyCompletion([search, { type: "assistant.refusal", data: { reason: "refusal" } }])).toBe(false);
+    expect(isCopilotToolSearchOnlyCompletion([])).toBe(false);
+  });
+
+  it.each(['{"type":"assistant.message"', '{"type":"assistant.mess', '{"type":"session.shut'])("reads only the exact native session and rejects truncated records (%s)", truncated => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "copilot-search-session-"));
+    tempDirs.push(root);
+    vi.stubEnv("COPILOT_HOME", root);
+    vi.stubEnv("GH_AW_TMP_DIR", root);
+    const sessionId = "aabbccdd-1122-3344-5566-778899aabbcc";
+    const dir = path.join(root, "session-state", sessionId);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(path.join(dir, "events.jsonl"), [{ type: "session.start" }, search, { type: "session.shutdown" }].map(event => JSON.stringify(event)).join("\n") + "\n");
+    expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", sessionId)).toBe(true);
+    expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", "11223344-1122-3344-5566-778899aabbcc")).toBe(false);
+    expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", "../other")).toBe(false);
+    expect(hasCopilotToolSearchOnlyCompletion(`${JSON.stringify({ type: "session.start" })}\n${JSON.stringify(search)}`, undefined)).toBe(true);
+    fs.appendFileSync(path.join(dir, "events.jsonl"), truncated);
+    expect(hasCopilotToolSearchOnlyCompletion("plain CLI output", sessionId)).toBe(false);
+  });
+
+  it.each([false, true])("native zero exit is rejected only without a terminal safe output (terminal=%s)", terminal => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "copilot-search-exit-"));
+    tempDirs.push(root);
+    const stub = path.join(root, "stub.cjs");
+    const safeOutputs = path.join(root, "safe-outputs.jsonl");
+    const configPath = path.join(root, "awf-config.json");
+    fs.writeFileSync(configPath, "{}");
+    fs.writeFileSync(safeOutputs, terminal ? '{"type":"submit_pull_request_review"}\n' : "");
+    fs.writeFileSync(
+      stub,
+      `const fs = require("fs"), path = require("path");
+const args = process.argv.slice(2);
+const id = args[args.indexOf("--session-id") + 1];
+const dir = path.join(process.env.COPILOT_HOME, "session-state", id);
+fs.mkdirSync(dir, { recursive: true });
+fs.writeFileSync(path.join(dir, "events.jsonl"), JSON.stringify(${JSON.stringify({ ...search, data: { ...search.data, model: "gpt-6.1-sol" } })}) + "\\n");
+process.stdout.write("Skill loaded successfully.\\n");`
+    );
+    const result = spawnSync(process.execPath, [require.resolve("./copilot_harness.cjs"), process.execPath, stub], {
+      cwd: root,
+      env: {
+        PATH: process.env.PATH,
+        HOME: root,
+        TMPDIR: root,
+        GITHUB_WORKSPACE: root,
+        GH_AW_ENGINE_CWD: root,
+        GH_AW_AWF_CONFIG_PATH: configPath,
+        GH_AW_HARNESS_INITIAL_DELAY_MS: "1",
+        GH_AW_HARNESS_MAX_DELAY_MS: "1",
+        COPILOT_HOME: root,
+        COPILOT_MODEL: "gpt-6.1-sol",
+        GH_AW_PHASE: "agent",
+        GH_AW_TMP_DIR: root,
+        GH_AW_SAFE_OUTPUTS: safeOutputs,
+        GH_AW_MODEL_ROUTING: "0",
+        AWF_REFLECT_ENABLED: "0",
+      },
+      encoding: "utf8",
+      timeout: 10000,
+    });
+    expect(result.status, result.stderr).toBe(terminal ? 0 : 1);
+    expect(result.stderr).toContain(terminal ? "success on attempt 1" : "Copilot stopped after server-side tool discovery");
+    expect(result.stderr).not.toContain("retry 1/");
+    expect(fs.readFileSync(safeOutputs, "utf8")).toBe(terminal ? '{"type":"submit_pull_request_review"}\n' : "");
+    if (!terminal) {
+      const retry = JSON.parse(fs.readFileSync(path.join(root, "agent/harness_retry.json"), "utf8"));
+      expect(retry.attempt_results[0].failure_classes).toEqual(["copilot_tool_search_only_completion"]);
+    }
+  });
+});
+
 const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard } = require("./harness_retry_guard.cjs");
 const { detectErrors } = require("./agent_error_patterns.cjs");
 

actions/setup/js/jsonl_helpers.cjs

@@ -8,6 +8,7 @@ const { prepareJsonSchema } = require("./json_schema_validation.cjs");
  * @typedef {object} JsonlOptions
  * @property {(line: string, lineNumber: number) => boolean} [lineFilter] Receives trimmed text before transformation and decoding.
  * @property {(line: string, lineNumber: number) => string} [lineTransform] Transforms trimmed record text after filtering, retaining the original rawLine.
+ * @property {(value: any, lineNumber: number) => boolean} [valueFilter] Selects decoded, schema-validated values without skipping invalid records.
  * @property {(error: Error, lineNumber: number) => void} [onError] Reports rejected nonblank lines without exposing their contents.
  * @property {number} [maxLineLength] Optional limit in UTF-16 code units; unlimited by default.
  * @property {{field: string, value: string|number|boolean|null}} [fieldFilter] Match an own top-level field, prefiltering text before decoding.
@@ -246,6 +247,7 @@ function parseEvent(rawLine, lineNumber, options, fieldFilter, schemaValidator)
     if (fieldFilter && (!value || typeof value !== "object" || Array.isArray(value) || !Object.hasOwn(value, fieldFilter.field) || value[fieldFilter.field] !== fieldFilter.value)) return;
     const validationError = schemaValidator?.(value, line);
     if (validationError) return invalidEvent(rawLine, lineNumber, "schema", options, validationError);
+    if (options.valueFilter && !options.valueFilter(value, lineNumber)) return;
     return { type: "json", value, rawLine, lineNumber };
   }
   return invalidEvent(rawLine, lineNumber, tooLong ? "too-long" : "syntax", options);

actions/setup/js/jsonl_helpers.test.cjs

@@ -14,6 +14,64 @@ async function collect(source) {
 }
 
 describe("jsonl_helpers", () => {
+  describe("parsed-value filtering", () => {
+    const valueFilter = value => value?.type === "wanted";
+
+    it("filters decoded values consistently across text, files, and streams", async () => {
+      const text = 'null\nfalse\n0\n{"nested":{"type":"wanted"}}\n{"type":"other"}\n{"\\u0074ype":"wanted","id":7}\n';
+      const expected = [{ type: "wanted", id: 7 }];
+      const directory = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-jsonl-value-filter-"));
+      const file = path.join(directory, "records.jsonl");
+      try {
+        fs.writeFileSync(file, text);
+        expect([...iterateJsonl(text, { valueFilter })]).toEqual(expected);
+        expect([...readJsonlFile(file, { valueFilter })]).toEqual(expected);
+        expect(await collect(streamJsonl([text], { valueFilter }))).toEqual(expected);
+      } finally {
+        fs.rmSync(directory, { recursive: true, force: true });
+      }
+    });
+
+    it("stops decoding and closes the file after the first parsed match", () => {
+      const directory = fs.mkdtempSync(path.join(os.tmpdir(), "gh-aw-jsonl-value-search-"));
+      const file = path.join(directory, "records.jsonl");
+      fs.writeFileSync(file, '{"type":"other"}\n{"type":"wanted"}\n{"broken":}\n');
+      const filter = vi.fn(valueFilter);
+      const parse = vi.spyOn(JSON, "parse");
+      const close = vi.spyOn(fs, "closeSync");
+      try {
+        for (const record of readJsonlFile(file, { strict: true, valueFilter: filter })) {
+          expect(record).toEqual({ type: "wanted" });
+          break;
+        }
+        expect(filter.mock.calls).toEqual([
+          [{ type: "other" }, 1],
+          [{ type: "wanted" }, 2],
+        ]);
+        expect(parse).toHaveBeenCalledTimes(2);
+        expect(close).toHaveBeenCalledTimes(1);
+      } finally {
+        parse.mockRestore();
+        close.mockRestore();
+        fs.rmSync(directory, { recursive: true, force: true });
+      }
+    });
+
+    it("preserves strict validation before filtering and propagates predicate errors", () => {
+      const filter = vi.fn(valueFilter);
+      expect(() => [...iterateJsonl('{"type":"wanted"}\n{"type":"oth', { strict: true, valueFilter: filter })]).toThrow(SyntaxError);
+      expect(filter).toHaveBeenCalledTimes(1);
+      const error = new Error("predicate failed");
+      expect(() => [
+        ...iterateJsonl("null", {
+          valueFilter: () => {
+            throw error;
+          },
+        }),
+      ]).toThrow(error);
+    });
+  });
+
   describe("field-value search", () => {
     const options = { fieldFilter: { field: "type", value: "wanted" } };