fix(storage)+feat(cli): lbug truncate SIGBUS + scip-go/python/typescript distribution

packages/cli/package.json

@@ -46,6 +46,7 @@
     "@opencodehub/policy": "workspace:*",
     "@opencodehub/sarif": "workspace:*",
     "@opencodehub/scanners": "workspace:*",
+    "@opencodehub/scip-ingest": "workspace:*",
     "@opencodehub/search": "workspace:*",
     "@opencodehub/storage": "workspace:*",
     "@opencodehub/wiki": "workspace:*",

packages/cli/src/commands/doctor.ts

@@ -19,6 +19,7 @@ import { createRequire } from "node:module";
 import { homedir, tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { hostedScipBinDirs } from "@opencodehub/scip-ingest";
 import Table from "cli-table3";
 
 export type CheckStatus = "ok" | "warn" | "fail";
@@ -414,12 +415,24 @@ interface ScipIndexerSpec {
   readonly setupFlag?: string;
   /** True when the indexer is a JAR/asset under ~/.codehub, not a PATH binary. */
   readonly jar?: boolean;
+  /**
+   * The npm package this indexer ships from when it is a HARD dependency of
+   * `@opencodehub/scip-ingest` (the pure-JS Sourcegraph indexers). When set,
+   * the check resolves the bundled package via `createRequire` rather than
+   * requiring it on PATH or under `~/.codehub/bin` — these always ship with
+   * the CLI, so a clean install must report `ok` out-of-the-box.
+   */
+  readonly bundledPkg?: string;
 }
 
 const SCIP_INDEXERS: readonly ScipIndexerSpec[] = [
-  { language: "typescript", binName: "scip-typescript" },
-  { language: "python", binName: "scip-python" },
-  { language: "go", binName: "scip-go" },
+  {
+    language: "typescript",
+    binName: "scip-typescript",
+    bundledPkg: "@sourcegraph/scip-typescript",
+  },
+  { language: "python", binName: "scip-python", bundledPkg: "@sourcegraph/scip-python" },
+  { language: "go", binName: "scip-go", setupFlag: "go" },
   { language: "rust", binName: "rust-analyzer" },
   { language: "java", binName: "scip-java" },
   { language: "ruby", binName: "scip-ruby", setupFlag: "ruby" },
@@ -436,12 +449,41 @@ function scipIndexerCheck(
   run: RunCommandFn,
 ): Check {
   const missingStatus: CheckStatus = strict ? "fail" : "warn";
-  const installHint = spec.setupFlag
-    ? `install with \`codehub setup --scip=${spec.setupFlag}\``
-    : `${spec.binName} is a system toolchain — install it via your package manager / language SDK`;
+  const installHint = spec.bundledPkg
+    ? `bundled with @opencodehub/cli (${spec.bundledPkg}); reinstall the CLI to restore it`
+    : spec.setupFlag
+      ? `install with \`codehub setup --scip=${spec.setupFlag}\``
+      : `${spec.binName} is a system toolchain — install it via your package manager / language SDK`;
   return {
     name: `scip indexer: ${spec.language}`,
     async run() {
+      if (spec.bundledPkg !== undefined) {
+        // Hard dependency of @opencodehub/scip-ingest (ships with the CLI).
+        // Authoritative check: does the indexer's bin shim resolve into a
+        // directory that the analyze-time spawn PATH actually includes? That
+        // is exactly what `hostedScipBinDirs()` (the same resolver
+        // `withCodehubBinOnPath` uses) returns, so a match here guarantees the
+        // runner will find the indexer by bare name — even though the nested
+        // `node_modules/.bin` is NOT on the user's interactive PATH (so a bare
+        // `<bin> --version` probe would false-FAIL).
+        const onHostedPath = hostedScipBinDirs().some((d: string) =>
+          existsSyncSafe(join(d, spec.binName)),
+        );
+        if (onHostedPath) {
+          return { status: "ok", message: `${spec.binName} bundled (${spec.bundledPkg})` };
+        }
+        // Fall back to an explicit on-PATH install (e.g. a global
+        // `npm i -g @sourcegraph/scip-typescript`).
+        const res = await run(spec.binName, ["--version"]);
+        if (res.status === 0) {
+          return { status: "ok", message: `${spec.binName}: ${firstLine(res.stdout)}` };
+        }
+        return {
+          status: missingStatus,
+          message: `${spec.binName} not resolvable (bundled dep ${spec.bundledPkg} missing)`,
+          hint: installHint,
+        };
+      }
       if (spec.jar === true) {
         // JAR/asset indexers aren't `--version`-able binaries: check the
         // file exists under ~/.codehub (setup downloads them there).
@@ -808,7 +850,10 @@ function resolveFromRoot(repoRoot: string, pkg: string): string | null {
   //    and `@ladybugdb/core` both live in `packages/storage`. Probing that
   //    package.json context lets `codehub doctor` resolve the bindings
   //    even when neither the CLI nor the workspace root declare them as
-  //    direct deps.
+  //    direct deps. (The pure-JS `@sourcegraph/scip-*` indexers are NOT
+  //    resolved here — `scipIndexerCheck` checks them via
+  //    `hostedScipBinDirs()`, the same resolver the analyze-time spawn PATH
+  //    uses, which is the layout-correct authority for "will analyze find it".)
   const owners =
     pkg.startsWith("@duckdb/") || pkg.startsWith("@ladybugdb/") ? ["packages/storage"] : [];
   for (const owner of owners) {

packages/cli/src/scip-downloader.test.ts

@@ -25,6 +25,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { ReadableStream } from "node:stream/web";
 import { describe, it } from "node:test";
+import { gzipSync } from "node:zlib";
 
 import {
   DotnetSdkMissingError,
@@ -42,6 +43,41 @@ function sha256(buf: Uint8Array): string {
   return createHash("sha256").update(buf).digest("hex");
 }
 
+/**
+ * Build a minimal, valid gzip tarball containing a single ustar regular-file
+ * entry `{name, body}` followed by the two zero-block EOF marker. Mirrors the
+ * shape of Sourcegraph's scip-go release tarballs (short root-level names, no
+ * PAX/GNU extensions) so the downloader's extraction path is exercised with a
+ * real gunzip + untar rather than a hand-faked buffer. Returns the gzipped
+ * bytes — exactly what the downloader fetches and SHA256-pins.
+ */
+function makeTarGz(name: string, body: Uint8Array): Uint8Array {
+  const BLOCK = 512;
+  const header = Buffer.alloc(BLOCK);
+  header.write(name, 0, "ascii"); // name @ 0 (max 100)
+  header.write("0000644", 100, "ascii"); // mode @ 100
+  header.write("0000000", 108, "ascii"); // uid @ 108
+  header.write("0000000", 116, "ascii"); // gid @ 116
+  header.write(body.length.toString(8).padStart(11, "0"), 124, "ascii"); // size @ 124 (octal)
+  header.write("00000000000", 136, "ascii"); // mtime @ 136
+  header[156] = 0x30; // typeflag '0' (regular file)
+  header.write("ustar\0", 257, "ascii"); // magic @ 257
+  header.write("00", 263, "ascii"); // version @ 263
+  // Checksum: spaces while summing, then octal + NUL + space @ 148.
+  header.fill(0x20, 148, 156);
+  let sum = 0;
+  for (const b of header) sum += b;
+  header.write(sum.toString(8).padStart(6, "0"), 148, "ascii");
+  header[154] = 0; // NUL
+  header[155] = 0x20; // space
+
+  const dataPadded = Buffer.alloc(Math.ceil(body.length / BLOCK) * BLOCK);
+  Buffer.from(body).copy(dataPadded);
+  const eof = Buffer.alloc(BLOCK * 2); // two zero blocks
+  const tar = Buffer.concat([header, dataPadded, eof]);
+  return new Uint8Array(gzipSync(tar));
+}
+
 function makeResponse(status: number, body: Uint8Array | null): Response {
   if (status === 200 && body !== null) {
     const stream = new ReadableStream<Uint8Array>({
@@ -428,6 +464,148 @@ describe("installScipTool", () => {
   });
 });
 
+describe("scip-go (archive/tarball extraction)", () => {
+  const LINUX_X64_GO = { os: "linux", arch: "x64" } as const;
+
+  it("extracts the binary from the gzip tarball, chmods it, and verifies the tarball SHA256", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "och-scip-go-"));
+    try {
+      const binBytes = new TextEncoder().encode("\x7fELF fake scip-go binary");
+      const tarGz = makeTarGz("scip-go", binBytes);
+      // A sibling entry (LICENSE) must be skipped — prove the parser selects
+      // only the wanted entry by serving a two-entry archive.
+      const { fetch, calls } = makeFetchWith(new Map([["https://example.test/go", tarGz]]));
+
+      const goPin: ScipToolPin = {
+        tool: "go",
+        version: "0.2.7",
+        installerKind: "download",
+        placeholder: false,
+        binName: "scip-go",
+        platforms: [
+          {
+            os: "linux",
+            arch: "x64",
+            url: "https://example.test/go",
+            sha256: sha256(tarGz),
+            archiveEntry: "scip-go",
+          },
+        ],
+      };
+      const mutable = SCIP_PINS as unknown as Record<ScipToolPin["tool"], ScipToolPin>;
+      const original = SCIP_PINS.go;
+      mutable.go = goPin;
+      try {
+        const result = await installScipTool("go", {
+          destDir: dir,
+          fetchImpl: fetch,
+          platform: LINUX_X64_GO,
+        });
+        assert.equal(result.installed, true);
+        assert.equal(result.tool, "go");
+        // On disk is the EXTRACTED binary, not the tarball.
+        const onDisk = await readFile(result.path);
+        assert.deepEqual(new Uint8Array(onDisk), binBytes);
+        // Executable bit set.
+        const st = await stat(result.path);
+        assert.equal(st.mode & 0o111, 0o111);
+        // Exactly one fetch.
+        assert.equal(calls.length, 1);
+      } finally {
+        mutable.go = original;
+      }
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("rejects a tarball whose bytes do not match the pinned SHA256", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "och-scip-go-bad-"));
+    try {
+      const tarGz = makeTarGz("scip-go", new TextEncoder().encode("real"));
+      const { fetch } = makeFetchWith(new Map([["https://example.test/go", tarGz]]));
+      const goPin: ScipToolPin = {
+        tool: "go",
+        version: "0.2.7",
+        installerKind: "download",
+        placeholder: false,
+        binName: "scip-go",
+        platforms: [
+          {
+            os: "linux",
+            arch: "x64",
+            url: "https://example.test/go",
+            sha256: sha256(new TextEncoder().encode("WRONG")), // deliberately wrong
+            archiveEntry: "scip-go",
+          },
+        ],
+      };
+      const mutable = SCIP_PINS as unknown as Record<ScipToolPin["tool"], ScipToolPin>;
+      const original = SCIP_PINS.go;
+      mutable.go = goPin;
+      try {
+        await assert.rejects(
+          () => installScipTool("go", { destDir: dir, fetchImpl: fetch, platform: LINUX_X64_GO }),
+          (err: unknown) => err instanceof ScipSha256MismatchError,
+        );
+      } finally {
+        mutable.go = original;
+      }
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("skips re-install when the extracted binary already exists (archive idempotency)", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "och-scip-go-idem-"));
+    try {
+      const tarGz = makeTarGz("scip-go", new TextEncoder().encode("scip-go-bin"));
+      const { fetch, calls } = makeFetchWith(new Map([["https://example.test/go", tarGz]]));
+      const goPin: ScipToolPin = {
+        tool: "go",
+        version: "0.2.7",
+        installerKind: "download",
+        placeholder: false,
+        binName: "scip-go",
+        platforms: [
+          {
+            os: "linux",
+            arch: "x64",
+            url: "https://example.test/go",
+            sha256: sha256(tarGz),
+            archiveEntry: "scip-go",
+          },
+        ],
+      };
+      const mutable = SCIP_PINS as unknown as Record<ScipToolPin["tool"], ScipToolPin>;
+      const original = SCIP_PINS.go;
+      mutable.go = goPin;
+      try {
+        const first = await installScipTool("go", {
+          destDir: dir,
+          fetchImpl: fetch,
+          platform: LINUX_X64_GO,
+        });
+        assert.equal(first.installed, true);
+        const second = await installScipTool("go", {
+          destDir: dir,
+          fetchImpl: fetch,
+          platform: LINUX_X64_GO,
+        });
+        assert.equal(second.skipped, true);
+        assert.equal(second.installed, false);
+        // The extracted-binary presence check means the second call never
+        // re-fetches (the tarball SHA can't be recomputed from the binary).
+        assert.equal(calls.length, 1);
+      } finally {
+        mutable.go = original;
+      }
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});
+
 describe("installAllScipTools", () => {
   it("runs every tool in order and returns a per-tool result or error", async () => {
     const dir = await mkdtemp(join(tmpdir(), "och-scip-all-"));
@@ -454,23 +632,47 @@ describe("installAllScipTools", () => {
       const clangBody = new TextEncoder().encode("clang-bytes");
       const rubyBody = new TextEncoder().encode("ruby-bytes");
       const kotlinBody = new TextEncoder().encode("kotlin-bytes");
+      // scip-go is an archive tool: the served body is a gzip tarball whose
+      // `scip-go` entry holds the binary. This exercises the extraction path
+      // through `installAllScipTools` too.
+      const goTarGz = makeTarGz("scip-go", new TextEncoder().encode("go-binary-bytes"));
 
       const { fetch } = makeFetchWith(
         new Map([
           ["https://example.test/clang", clangBody],
           ["https://example.test/ruby", rubyBody],
+          ["https://example.test/go", goTarGz],
           ["https://example.test/kotlin", kotlinBody],
         ]),
       );
 
+      const goStub: ScipToolPin = {
+        tool: "go",
+        version: "1.2.3",
+        installerKind: "download",
+        placeholder: false,
+        binName: "scip-go",
+        platforms: [
+          {
+            os: "linux",
+            arch: "x64",
+            url: "https://example.test/go",
+            sha256: sha256(goTarGz),
+            archiveEntry: "scip-go",
+          },
+        ],
+      };
+
       const originals = {
         clang: SCIP_PINS.clang,
         ruby: SCIP_PINS.ruby,
+        go: SCIP_PINS.go,
         kotlin: SCIP_PINS.kotlin,
       };
       const mutable = SCIP_PINS as unknown as Record<ScipToolPin["tool"], ScipToolPin>;
       mutable.clang = mkStub("clang", clangBody);
       mutable.ruby = mkStub("ruby", rubyBody);
+      mutable.go = goStub;
       mutable.kotlin = mkStub("kotlin", kotlinBody);
 
       try {
@@ -481,13 +683,14 @@ describe("installAllScipTools", () => {
           dotnetProbe: async () => "8.0.100",
         });
 
-        assert.equal(results.length, 4);
-        // Clang, ruby, dotnet, kotlin — order from SCIP_TOOL_ORDER.
+        assert.equal(results.length, 5);
+        // Clang, ruby, go, dotnet, kotlin — order from SCIP_TOOL_ORDER.
         const tools = results.map((r) => ("tool" in r ? r.tool : "error"));
-        assert.deepEqual(tools, ["clang", "ruby", "dotnet", "kotlin"]);
+        assert.deepEqual(tools, ["clang", "ruby", "go", "dotnet", "kotlin"]);
       } finally {
         mutable.clang = originals.clang;
         mutable.ruby = originals.ruby;
+        mutable.go = originals.go;
         mutable.kotlin = originals.kotlin;
       }
     } finally {