diff --git a/lib/entry-points.js b/lib/entry-points.js index cbeec24780..f78a951f06 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -34699,7 +34699,7 @@ var require_cacheUtils = __commonJS({ var crypto3 = __importStar2(require("crypto")); var fs32 = __importStar2(require("fs")); var path30 = __importStar2(require("path")); - var semver12 = __importStar2(require_semver3()); + var semver13 = __importStar2(require_semver3()); var util3 = __importStar2(require("util")); var constants_1 = require_constants7(); var versionSalt = "1.0"; @@ -34792,7 +34792,7 @@ var require_cacheUtils = __commonJS({ function getCompressionMethod() { return __awaiter2(this, void 0, void 0, function* () { const versionOutput = yield getVersion("zstd", ["--quiet"]); - const version = semver12.clean(versionOutput); + const version = semver13.clean(versionOutput); core32.debug(`zstd version: ${version}`); if (versionOutput === "") { return constants_1.CompressionMethod.Gzip; @@ -82401,7 +82401,7 @@ var require_manifest = __commonJS({ exports2._findMatch = _findMatch; exports2._getOsVersion = _getOsVersion; exports2._readLinuxVersionFile = _readLinuxVersionFile; - var semver12 = __importStar2(require_semver2()); + var semver13 = __importStar2(require_semver2()); var core_1 = require_core(); var os7 = require("os"); var cp = require("child_process"); @@ -82415,7 +82415,7 @@ var require_manifest = __commonJS({ for (const candidate of candidates) { const version = candidate.version; (0, core_1.debug)(`check ${version} satisfies ${versionSpec}`); - if (semver12.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { + if (semver13.satisfies(version, versionSpec) && (!stable || candidate.stable === stable)) { file = candidate.files.find((item) => { (0, core_1.debug)(`${item.arch}===${archFilter} && ${item.platform}===${platFilter}`); let chk = item.arch === archFilter && item.platform === platFilter; @@ -82424,7 +82424,7 @@ var require_manifest = __commonJS({ if (osVersion === item.platform_version) { chk = true; } else { - chk = semver12.satisfies(osVersion, item.platform_version); + chk = semver13.satisfies(osVersion, item.platform_version); } } return chk; @@ -82684,7 +82684,7 @@ var require_tool_cache = __commonJS({ var os7 = __importStar2(require("os")); var path30 = __importStar2(require("path")); var httpm = __importStar2(require_lib()); - var semver12 = __importStar2(require_semver2()); + var semver13 = __importStar2(require_semver2()); var stream2 = __importStar2(require("stream")); var util3 = __importStar2(require("util")); var assert_1 = require("assert"); @@ -82957,7 +82957,7 @@ var require_tool_cache = __commonJS({ } function cacheDir2(sourceDir, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver12.clean(version) || version; + version = semver13.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source dir: ${sourceDir}`); @@ -82975,7 +82975,7 @@ var require_tool_cache = __commonJS({ } function cacheFile(sourceFile, targetFile, tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - version = semver12.clean(version) || version; + version = semver13.clean(version) || version; arch2 = arch2 || os7.arch(); core32.debug(`Caching tool ${tool} ${version} ${arch2}`); core32.debug(`source file: ${sourceFile}`); @@ -83005,7 +83005,7 @@ var require_tool_cache = __commonJS({ } let toolPath = ""; if (versionSpec) { - versionSpec = semver12.clean(versionSpec) || ""; + versionSpec = semver13.clean(versionSpec) || ""; const cachePath = path30.join(_getCacheDirectory(), toolName, versionSpec, arch2); core32.debug(`checking cache: ${cachePath}`); if (fs32.existsSync(cachePath) && fs32.existsSync(`${cachePath}.complete`)) { @@ -83085,7 +83085,7 @@ var require_tool_cache = __commonJS({ } function _createToolPath(tool, version, arch2) { return __awaiter2(this, void 0, void 0, function* () { - const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver13.clean(version) || version, arch2 || ""); core32.debug(`destination ${folderPath}`); const markerPath = `${folderPath}.complete`; yield io9.rmRF(folderPath); @@ -83095,15 +83095,15 @@ var require_tool_cache = __commonJS({ }); } function _completeToolPath(tool, version, arch2) { - const folderPath = path30.join(_getCacheDirectory(), tool, semver12.clean(version) || version, arch2 || ""); + const folderPath = path30.join(_getCacheDirectory(), tool, semver13.clean(version) || version, arch2 || ""); const markerPath = `${folderPath}.complete`; fs32.writeFileSync(markerPath, ""); core32.debug("finished caching tool"); } function isExplicitVersion(versionSpec) { - const c = semver12.clean(versionSpec) || ""; + const c = semver13.clean(versionSpec) || ""; core32.debug(`isExplicit: ${c}`); - const valid4 = semver12.valid(c) != null; + const valid4 = semver13.valid(c) != null; core32.debug(`explicit? ${valid4}`); return valid4; } @@ -83111,14 +83111,14 @@ var require_tool_cache = __commonJS({ let version = ""; core32.debug(`evaluating ${versions.length} versions`); versions = versions.sort((a, b) => { - if (semver12.gt(a, b)) { + if (semver13.gt(a, b)) { return 1; } return -1; }); for (let i = versions.length - 1; i >= 0; i--) { const potential = versions[i]; - const satisfied = semver12.satisfies(potential, versionSpec); + const satisfied = semver13.satisfies(potential, versionSpec); if (satisfied) { version = potential; break; @@ -151221,7 +151221,7 @@ var import_perf_hooks4 = require("perf_hooks"); var core12 = __toESM(require_core()); var toolcache3 = __toESM(require_tool_cache()); var import_fast_deep_equal = __toESM(require_fast_deep_equal()); -var semver10 = __toESM(require_semver2()); +var semver11 = __toESM(require_semver2()); // src/codeql-bundle.ts function getCodeQLBundleName(compressionMethod, platform2, language) { @@ -151252,10 +151252,200 @@ function getCodeQLBundleFromUrl(url2) { return language === void 0 ? { kind: "combined", url: url2 } : { kind: "per-language", url: url2, language }; } +// src/codeql-release.ts +var semver7 = __toESM(require_semver2()); + +// src/per-language-bundles.ts +var semver6 = __toESM(require_semver2()); + +// src/platform.ts +function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { + switch (platform2) { + case "win32": + return "win64" /* Win64 */; + case "linux": + return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; + case "darwin": + return "osx64" /* Osx64 */; + default: + return void 0; + } +} + +// src/per-language-bundles.ts +var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; +var PER_LANGUAGE_BUNDLE_LANGUAGES = { + ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ + "actions" /* actions */, + "cpp" /* cpp */, + "csharp" /* csharp */, + "go" /* go */, + "java" /* java */, + "javascript" /* javascript */, + "python" /* python */, + "ruby" /* ruby */, + "rust" /* rust */ + ]), + ["linux-arm64" /* LinuxArm64 */]: /* @__PURE__ */ new Set(), + ["osx64" /* Osx64 */]: /* @__PURE__ */ new Set(["swift" /* swift */]), + ["win64" /* Win64 */]: /* @__PURE__ */ new Set() +}; +async function getPerLanguageBundleLanguage({ + env, + features, + logger +}, options) { + const { + rawLanguages, + cliVersion: cliVersion2, + compressionMethod, + platform: platform2, + variant, + isLatestNightly + } = options; + const explain = (reason) => { + logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); + return void 0; + }; + if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { + return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); + } + if (rawLanguages?.length !== 1) { + return explain( + `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` + ); + } + const language = parseBuiltInLanguage(rawLanguages[0]); + if (language === void 0) { + return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); + } + if (compressionMethod !== "zstd") { + return explain(`the bundle would be downloaded as '${compressionMethod}'`); + } + if (variant !== "GitHub.com" /* DOTCOM */) { + return explain(`we are running against ${variant}`); + } + if (!isGitHubHostedRunner(env)) { + return explain("the job is not running on a GitHub-hosted runner"); + } + if (!isLatestNightly) { + if (cliVersion2 === void 0) { + return explain("the requested CLI version is unknown"); + } + if (!semver6.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { + return explain( + `the requested CodeQL version ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version for which per-language bundles are published` + ); + } + } + if (platform2 === void 0 || !PER_LANGUAGE_BUNDLE_LANGUAGES[platform2].has(language)) { + return explain( + `no per-language bundle is published for ${language} on ${platform2 ?? "an unknown platform"}` + ); + } + return language; +} +function logMissingPerLanguageBundle({ logger }, language, location) { + logger.warning( + `Expected a per-language CodeQL bundle for '${language}' at ${location}, but it wasn't found, so using the bundle that contains all languages instead. This analysis will still produce correct results, but will take longer to set up.` + ); +} + +// src/codeql-release.ts +function encodeTag(tagName) { + return tagName.split("/").map(encodeURIComponent).join("/"); +} +function getReleasePageURL(reference) { + const { serverURL, owner, repo, tagName } = reference; + return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`; +} +async function getRelease({ apiClient }, reference) { + const { owner, repo, tagName } = reference; + const { data: release2 } = await apiClient.rest.repos.getReleaseByTag({ + owner, + repo, + tag: tagName + }); + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => release2.assets.find((asset) => asset.name === name)?.url + }; +} +function getPublicRelease(reference) { + const { serverURL, owner, repo, tagName } = reference; + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => `${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}` + }; +} +function getCompressionMethods({ + cliVersion: cliVersion2, + isLatestNightly, + platform: platform2, + tarSupportsZstd +}) { + if (!tarSupportsZstd) { + return ["gzip"]; + } + const preferZstd = ( + // In testing, gzip performs better than zstd on Windows. + platform2 !== "win64" /* Win64 */ && // Standard bundles have zstd archives from this version, and so does the latest nightly. For a + // release we looked up, gzip comes next if it lacks the zstd archive. + (isLatestNightly || cliVersion2 !== void 0 && semver7.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE)) + ); + return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"]; +} +async function selectBundle(action, release2, options) { + const { logger } = action; + const compressionMethods = getCompressionMethods(options); + for (const compressionMethod of compressionMethods) { + const combinedBundleName = getCodeQLBundleName( + compressionMethod, + options.platform + ); + const combinedBundleURL = release2.getAssetURL(combinedBundleName); + if (combinedBundleURL === void 0) { + continue; + } + const combined = { + bundle: { kind: "combined", url: combinedBundleURL }, + compressionMethod + }; + const language = await getPerLanguageBundleLanguage(action, { + ...options, + compressionMethod + }); + if (language === void 0) { + logger.info( + `Selected CodeQL bundle ${combinedBundleName} from ${release2.url}.` + ); + return combined; + } + const name = getCodeQLBundleName( + compressionMethod, + options.platform, + language + ); + const url2 = release2.getAssetURL(name); + if (url2 === void 0) { + logMissingPerLanguageBundle(action, language, release2.url); + return { ...combined, perLanguageBundleFallback: true }; + } + logger.info(`Selected CodeQL bundle ${name} from ${release2.url}.`); + return { + bundle: { kind: "per-language", url: url2, language, combinedBundleURL }, + compressionMethod + }; + } + throw new ConfigurationError( + `No compatible CodeQL bundle was found in release ${release2.url}. Expected ${compressionMethods.map((method) => getCodeQLBundleName(method, options.platform)).join(" or ")}.` + ); +} + // src/overlay/caching.ts var fs11 = __toESM(require("fs")); var actionsCache3 = __toESM(require_cache4()); -var semver6 = __toESM(require_semver2()); +var semver8 = __toESM(require_semver2()); var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 7500; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; var CACHE_VERSION2 = 1; @@ -151532,7 +151722,7 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { if (!cache.key) continue; const suffix = cache.key.substring(cacheKeyPrefix.length); const match2 = suffix.match(versionRegex); - if (match2 && semver6.valid(match2[1])) { + if (match2 && semver8.valid(match2[1])) { versionSet.add(match2[1]); } } @@ -151542,109 +151732,13 @@ async function getCodeQlVersionsForOverlayBaseDatabases(rawLanguages, logger) { ); return []; } - const versions = [...versionSet].sort(semver6.rcompare); + const versions = [...versionSet].sort(semver8.rcompare); logger.info( `Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}` ); return versions; } -// src/per-language-bundles.ts -var semver7 = __toESM(require_semver2()); - -// src/platform.ts -function getBundlePlatform(platform2 = process.platform, arch2 = process.arch) { - switch (platform2) { - case "win32": - return "win64" /* Win64 */; - case "linux": - return arch2 === "arm64" ? "linux-arm64" /* LinuxArm64 */ : "linux64" /* Linux64 */; - case "darwin": - return "osx64" /* Osx64 */; - default: - return void 0; - } -} - -// src/per-language-bundles.ts -var MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION = "2.27.1"; -var PER_LANGUAGE_BUNDLE_LANGUAGES = { - ["linux64" /* Linux64 */]: /* @__PURE__ */ new Set([ - "actions" /* actions */, - "cpp" /* cpp */, - "csharp" /* csharp */, - "go" /* go */, - "java" /* java */, - "javascript" /* javascript */, - "python" /* python */, - "ruby" /* ruby */, - "rust" /* rust */ - ]), - ["linux-arm64" /* LinuxArm64 */]: /* @__PURE__ */ new Set(), - ["osx64" /* Osx64 */]: /* @__PURE__ */ new Set(["swift" /* swift */]), - ["win64" /* Win64 */]: /* @__PURE__ */ new Set() -}; -async function getPerLanguageBundleLanguage({ - env, - features, - logger -}, options) { - const { - rawLanguages, - cliVersion: cliVersion2, - compressionMethod, - platform: platform2, - variant, - isLatestNightly - } = options; - const explain = (reason) => { - logger.debug(`Not using a per-language CodeQL bundle since ${reason}.`); - return void 0; - }; - if (!await features.getValue("per_language_bundles" /* PerLanguageBundles */)) { - return explain(`the ${"per_language_bundles" /* PerLanguageBundles */} feature is disabled`); - } - if (rawLanguages?.length !== 1) { - return explain( - `exactly one language must be requested via the 'languages' input, but ${rawLanguages?.length ?? 0} were` - ); - } - const language = parseBuiltInLanguage(rawLanguages[0]); - if (language === void 0) { - return explain(`'${rawLanguages[0]}' is not a known CodeQL language`); - } - if (compressionMethod !== "zstd") { - return explain(`the bundle would be downloaded as '${compressionMethod}'`); - } - if (variant !== "GitHub.com" /* DOTCOM */) { - return explain(`we are running against ${variant}`); - } - if (!isGitHubHostedRunner(env)) { - return explain("the job is not running on a GitHub-hosted runner"); - } - if (!isLatestNightly) { - if (cliVersion2 === void 0) { - return explain("the requested CLI version is unknown"); - } - if (!semver7.gte(cliVersion2, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION)) { - return explain( - `the requested CodeQL version ${cliVersion2} is older than ${MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION}, which is the first version for which per-language bundles are published` - ); - } - } - if (platform2 === void 0 || !PER_LANGUAGE_BUNDLE_LANGUAGES[platform2].has(language)) { - return explain( - `no per-language bundle is published for ${language} on ${platform2 ?? "an unknown platform"}` - ); - } - return language; -} -function logPerLanguageBundleFallback({ logger }, language, location) { - logger.warning( - `No per-language CodeQL bundle for '${language}' was found at ${location}, so falling back to the bundle that contains all languages. This analysis will still produce correct results, but will take longer to set up.` - ); -} - // src/tar.ts var import_child_process = require("child_process"); var fs12 = __toESM(require("fs")); @@ -151652,7 +151746,7 @@ var stream = __toESM(require("stream")); var import_toolrunner = __toESM(require_toolrunner()); var io4 = __toESM(require_io()); var toolcache = __toESM(require_tool_cache()); -var semver8 = __toESM(require_semver2()); +var semver9 = __toESM(require_semver2()); var MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3"; var MIN_REQUIRED_GNU_TAR_VERSION = "1.31"; async function getTarVersion() { @@ -151694,9 +151788,9 @@ async function isZstdAvailable(logger) { case "gnu": return { available: foundZstdBinary && // GNU tar only uses major and minor version numbers - semver8.gte( - semver8.coerce(version), - semver8.coerce(MIN_REQUIRED_GNU_TAR_VERSION) + semver9.gte( + semver9.coerce(version), + semver9.coerce(MIN_REQUIRED_GNU_TAR_VERSION) ), foundZstdBinary, version: tarVersion @@ -151705,7 +151799,7 @@ async function isZstdAvailable(logger) { return { available: foundZstdBinary && // Do a loose comparison since these version numbers don't contain // a patch version number. - semver8.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), + semver9.gte(version, MIN_REQUIRED_BSD_TAR_VERSION), foundZstdBinary, version: tarVersion }; @@ -151814,7 +151908,7 @@ var core11 = __toESM(require_core()); var import_http_client = __toESM(require_lib()); var toolcache2 = __toESM(require_tool_cache()); var import_follow_redirects = __toESM(require_follow_redirects()); -var semver9 = __toESM(require_semver2()); +var semver10 = __toESM(require_semver2()); var STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; var STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1e3; var TOOLCACHE_TOOL_NAME = "CodeQL"; @@ -151940,7 +152034,7 @@ function getToolcacheToolDirectory(env) { ); } function getToolcacheVersionDirectoryName(version) { - return semver9.clean(version) || version; + return semver10.clean(version) || version; } function getToolcacheDirectory(version) { return path12.join( @@ -152051,7 +152145,8 @@ function getCodeQLActionRepository(logger) { } return getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, logger) { +async function selectDefaultBundle(action, tagName, apiDetails, options) { + const { logger } = action; const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -152066,33 +152161,34 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, codeQLBundleName, return !self2.slice(0, index2).some((other) => (0, import_fast_deep_equal.default)(source, other)); } ); - for (const downloadSource of uniqueDownloadSources) { - const [apiURL, repository] = downloadSource; - if (apiURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { + for (const [serverURL, repository] of uniqueDownloadSources) { + if (serverURL === GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY) { break; } - const [repositoryOwner, repositoryName] = repository.split("/"); + const [owner2, repo2] = repository.split("/"); try { - const release2 = await getApiClient().rest.repos.getReleaseByTag({ - owner: repositoryOwner, - repo: repositoryName, - tag: tagName - }); - for (const asset of release2.data.assets) { - if (asset.name === codeQLBundleName) { - logger.info( - `Found CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} with URL ${asset.url}.` - ); - return asset.url; - } - } + const release2 = await getRelease( + { apiClient: getApiClient() }, + { serverURL, owner: owner2, repo: repo2, tagName } + ); + return await selectBundle(action, release2, options); } catch (e) { logger.info( - `Looked for CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} but got error ${e}.` + `Looked for CodeQL bundles in release ${tagName} of ${repository} on ${serverURL} but got error ${e}.` ); } } - return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; + const [owner, repo] = CODEQL_DEFAULT_ACTION_REPOSITORY.split("/"); + return selectBundle( + action, + getPublicRelease({ + serverURL: GITHUB_DOTCOM_URL, + owner, + repo, + tagName + }), + options + ); } function tryGetBundleVersionFromTagName(tagName, logger) { const match2 = tagName.match(/^codeql-bundle-(.+)$/); @@ -152120,13 +152216,13 @@ function tryGetTagNameFromUrl(url2, logger) { return match2[1]; } function convertToSemVer(version, logger) { - if (!semver10.valid(version)) { + if (!semver11.valid(version)) { logger.debug( `Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.` ); version = `0.0.0-${version}`; } - const s = semver10.clean(version); + const s = semver11.clean(version); if (!s) { throw new Error(`Bundle version ${version} is not in SemVer format.`); } @@ -152288,7 +152384,8 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO bundle = await getLatestNightlyBundle( { env: getEnv(), features, logger }, rawLanguages, - variant + variant, + tarSupportsZstd ); toolsInput = bundle.url; } @@ -152341,7 +152438,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO url2 = toolsInput; if (tagName) { const bundleVersion3 = tryGetBundleVersionFromTagName(tagName, logger); - if (bundleVersion3 !== void 0 && semver10.valid(bundleVersion3)) { + if (bundleVersion3 !== void 0 && semver11.valid(bundleVersion3)) { cliVersion2 = convertToSemVer(bundleVersion3, logger); } } @@ -152442,47 +152539,26 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO } } let compressionMethod; + let perLanguageBundleFallback; if (!url2) { - const bundleTagName = tagName; - if (bundleTagName === void 0) { + if (tagName === void 0) { throw new Error( "Could not determine a release tag for the requested CodeQL bundle." ); } - compressionMethod = cliVersion2 !== void 0 && await useZstdBundle(cliVersion2, tarSupportsZstd) ? "zstd" : "gzip"; - const platform2 = getBundlePlatform(); - const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( + ({ bundle, compressionMethod, perLanguageBundleFallback } = await selectDefaultBundle( { env: getEnv(), features, logger }, + tagName, + apiDetails, { rawLanguages, cliVersion: cliVersion2, - compressionMethod, - platform: platform2, - variant + platform: getBundlePlatform(), + variant, + tarSupportsZstd } - ); - const resolveBundleURL = (language) => getCodeQLBundleDownloadURL( - bundleTagName, - apiDetails, - getCodeQLBundleName(compressionMethod, platform2, language), - logger - ); - const combinedBundleURL = await resolveBundleURL(); - if (perLanguageBundleLanguage !== void 0) { - logger.info( - `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.` - ); - url2 = await resolveBundleURL(perLanguageBundleLanguage); - bundle = { - kind: "per-language", - url: url2, - language: perLanguageBundleLanguage, - combinedBundleURL - }; - } else { - url2 = combinedBundleURL; - bundle = { kind: "combined", url: url2 }; - } + )); + url2 = bundle.url; } else { const method = inferCompressionMethod(url2); if (method === void 0) { @@ -152508,6 +152584,7 @@ async function getCodeQLSource(toolsInput, defaultCliVersion, rawLanguages, useO bundleVersion: bundleVersion2, cliVersion: cliVersion2, compressionMethod, + ...perLanguageBundleFallback ? { perLanguageBundleFallback } : {}, sourceType: "download", toolsVersion: resolvedVersion ?? "unknown" }; @@ -152564,6 +152641,9 @@ var downloadCodeQL = async function(source, apiDetails, tarVersion, tempDir, log statusReport: bundle.kind === "per-language" ? { ...statusReport, perLanguage: { tools_bundle_language: bundle.language } + } : source.perLanguageBundleFallback ? { + ...statusReport, + perLanguage: { tools_per_language_bundle_fallback: true } } : statusReport }; }; @@ -152697,11 +152777,12 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp if (bundle.kind !== "per-language" || bundle.combinedBundleURL === void 0 || asHTTPError(e)?.status !== 404) { throw e; } - logPerLanguageBundleFallback(action, bundle.language, bundle.url); + logMissingPerLanguageBundle(action, bundle.language, bundle.url); const result = await downloadCodeQL( { ...source, - bundle: { kind: "combined", url: bundle.combinedBundleURL } + bundle: { kind: "combined", url: bundle.combinedBundleURL }, + perLanguageBundleFallback: true }, apiDetails, tarVersion, @@ -152712,37 +152793,16 @@ async function downloadCodeQLBundle(action, source, apiDetails, tarVersion, temp ...result, statusReport: { ...result.statusReport, - totalDurationMs: durationMsSince(startTime), - perLanguage: { tools_per_language_bundle_fallback: true } + totalDurationMs: durationMsSince(startTime) } }; } } -async function useZstdBundle(cliVersion2, tarSupportsZstd) { - return ( - // In testing, gzip performs better than zstd on Windows. - process.platform !== "win32" && tarSupportsZstd && semver10.gte(cliVersion2, CODEQL_VERSION_ZSTD_BUNDLE) - ); -} function getTempExtractionDir(tempDir) { return path13.join(tempDir, v4_default()); } -async function getLatestNightlyBundle(action, rawLanguages, variant) { - const { logger } = action; - const zstdAvailability = await isZstdAvailable(logger); - const compressionMethod = await useZstdBundle( - CODEQL_VERSION_ZSTD_BUNDLE, - zstdAvailability.available - ) ? "zstd" : "gzip"; - const platform2 = getBundlePlatform(); - const language = await getPerLanguageBundleLanguage(action, { - rawLanguages, - cliVersion: void 0, - compressionMethod, - platform: platform2, - variant, - isLatestNightly: true - }); +async function getLatestNightlyBundle(action, rawLanguages, variant, tarSupportsZstd) { + let tagName; try { const release2 = await getApiClient().rest.repos.listReleases({ owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, @@ -152755,26 +152815,33 @@ async function getLatestNightlyBundle(action, rawLanguages, variant) { if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - const assetUrl = (name) => `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; - const url2 = assetUrl( - getCodeQLBundleName(compressionMethod, platform2, language) - ); - return language === void 0 ? { kind: "combined", url: url2 } : { - kind: "per-language", - url: url2, - language, - combinedBundleURL: assetUrl( - getCodeQLBundleName(compressionMethod, platform2) - ) - }; + tagName = latestRelease.tag_name; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${wrapError(e)}` ); } + const { bundle } = await selectBundle( + action, + getPublicRelease({ + serverURL: GITHUB_DOTCOM_URL, + owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, + repo: CODEQL_NIGHTLIES_REPOSITORY_NAME, + tagName + }), + { + rawLanguages, + cliVersion: void 0, + platform: getBundlePlatform(), + variant, + tarSupportsZstd, + isLatestNightly: true + } + ); + return bundle; } function getLatestToolcacheVersion(logger) { - const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver10.compare(b, a)); + const allVersions = toolcache3.findAllVersions("CodeQL").sort((a, b) => semver11.compare(b, a)); logger.debug( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( allVersions @@ -158917,7 +158984,7 @@ var import_async = __toESM(require_async(), 1); var import_path7 = require("path"); // node_modules/archiver/lib/error.js -var import_util36 = __toESM(require("util"), 1); +var import_util37 = __toESM(require("util"), 1); var ERROR_CODES = { ABORTED: "archive was aborted", DIRECTORYDIRPATHREQUIRED: "diretory dirpath argument must be a non-empty string value", @@ -158942,7 +159009,7 @@ function ArchiverError(code, data) { this.code = code; this.data = data; } -import_util36.default.inherits(ArchiverError, Error); +import_util37.default.inherits(ArchiverError, Error); // node_modules/archiver/lib/core.js var import_readable_stream2 = __toESM(require_ours(), 1); @@ -161874,7 +161941,7 @@ var fs29 = __toESM(require("fs")); var path25 = __toESM(require("path")); var core22 = __toESM(require_core()); var io7 = __toESM(require_io()); -var semver11 = __toESM(require_semver2()); +var semver12 = __toESM(require_semver2()); // src/config/inputs.ts async function getToolsInput(action, repositoryProperties) { @@ -162363,12 +162430,12 @@ async function run3(actionState) { const experimental = "2.19.3"; const publicPreview = "2.22.1"; const actualVer = (await codeql.getVersion()).version; - if (semver11.lt(actualVer, experimental)) { + if (semver12.lt(actualVer, experimental)) { throw new ConfigurationError( `Rust analysis is supported by CodeQL CLI version ${experimental} or higher, but found version ${actualVer}` ); } - if (semver11.lt(actualVer, publicPreview)) { + if (semver12.lt(actualVer, publicPreview)) { core22.exportVariable("CODEQL_ENABLE_EXPERIMENTAL_FEATURES" /* EXPERIMENTAL_FEATURES */, "true"); logger.info("Experimental Rust analysis enabled"); } diff --git a/src/codeql-bundle.ts b/src/codeql-bundle.ts index 2d262c848e..99481834df 100644 --- a/src/codeql-bundle.ts +++ b/src/codeql-bundle.ts @@ -27,6 +27,8 @@ export interface CodeQLDownloadSource { cliVersion?: string; /** Resolved version for telemetry, independent of whether the bundle can be cached. */ toolsVersion: string; + /** The release lacks the eligible per-language bundle, so we selected the combined bundle. */ + perLanguageBundleFallback?: true; } /** Returns the exact bundle asset name for a platform and optional language. */ diff --git a/src/codeql-release.test.ts b/src/codeql-release.test.ts new file mode 100644 index 0000000000..ce29a4bf13 --- /dev/null +++ b/src/codeql-release.test.ts @@ -0,0 +1,200 @@ +import * as github from "@actions/github"; +import test from "ava"; + +import { + BundleSelectionOptions, + getPublicRelease, + getRelease, + selectBundle, +} from "./codeql-release"; +import { ActionsEnvVars } from "./environment"; +import { Feature } from "./feature-flags"; +import { BuiltInLanguage } from "./languages"; +import { BundlePlatform } from "./platform"; +import { + createFeatures, + getRecordingLogger, + getTestEnv, + initAllState, + LoggedMessage, +} from "./testing-utils"; +import { ConfigurationError, GitHubVariant } from "./util"; + +const TAG = "codeql-bundle-v2.27.1"; +const REFERENCE = { + serverURL: "https://github.com", + owner: "octo", + repo: "tools", + tagName: TAG, +}; +const RELEASE_PAGE = `https://github.com/octo/tools/releases/tag/${TAG}`; +const API_URL = "https://api.github.com"; +const COMBINED = "codeql-bundle-linux64.tar.zst"; +const JAVA = "codeql-bundle-java-linux64.tar.zst"; + +const OPTIONS: BundleSelectionOptions = { + rawLanguages: ["java-kotlin"], + cliVersion: "2.27.1", + platform: BundlePlatform.Linux64, + variant: GitHubVariant.DOTCOM, + tarSupportsZstd: true, +}; + +/** Serves the release tagged `TAG` with the given assets from a stubbed API. */ +function releaseFixture({ assetNames = [COMBINED, JAVA], status = 200 } = {}) { + const apiBase = `${API_URL}/repos/octo/tools/releases`; + const releaseAPIURL = `${apiBase}/tags/${TAG}`; + const assets = assetNames.map((name, index) => ({ + name, + url: `${apiBase}/assets/${1000 + index}`, + })); + const requests: string[] = []; + const messages: LoggedMessage[] = []; + const state = initAllState({ + env: getTestEnv({ [ActionsEnvVars.RUNNER_ENVIRONMENT]: "github-hosted" }), + logger: getRecordingLogger(messages, { logToConsole: false }), + features: createFeatures([Feature.PerLanguageBundles]), + apiClient: github.getOctokit("123", { + baseUrl: API_URL, + request: { + fetch: async (url) => { + requests.push(String(url)); + if (String(url) !== releaseAPIURL) { + throw new Error(`Unexpected API request: ${url}`); + } + return new Response(JSON.stringify({ tag_name: TAG, assets }), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }, + }), + }); + return { + assets, + messages, + releaseAPIURL, + requests, + state, + select: async (options: Partial = {}) => + selectBundle(state, await getRelease(state, REFERENCE), { + ...OPTIONS, + ...options, + }), + }; +} + +test("selectBundle selects an eligible per-language bundle with a single release lookup", async (t) => { + const fixture = releaseFixture(); + t.deepEqual(await fixture.select(), { + bundle: { + kind: "per-language", + url: fixture.assets[1].url, + language: BuiltInLanguage.java, + combinedBundleURL: fixture.assets[0].url, + }, + compressionMethod: "zstd", + }); + t.deepEqual(fixture.requests, [fixture.releaseAPIURL]); +}); + +test("selectBundle falls back to the combined bundle from the same release", async (t) => { + const fixture = releaseFixture({ assetNames: [COMBINED] }); + t.deepEqual(await fixture.select(), { + bundle: { kind: "combined", url: fixture.assets[0].url }, + compressionMethod: "zstd", + perLanguageBundleFallback: true, + }); + t.true( + fixture.messages.some( + (message) => + message.type === "warning" && + typeof message.message === "string" && + message.message.includes(`'java' at ${RELEASE_PAGE}`), + ), + ); +}); + +test("selectBundle selects the combined bundle for jobs that aren't eligible for a per-language bundle", async (t) => { + const fixture = releaseFixture(); + t.deepEqual(await fixture.select({ rawLanguages: ["java", "python"] }), { + bundle: { kind: "combined", url: fixture.assets[0].url }, + compressionMethod: "zstd", + }); +}); + +test("selectBundle uses the other compression method when the preferred one is missing", async (t) => { + const gzipOnly = releaseFixture({ + assetNames: ["codeql-bundle-linux64.tar.gz", JAVA], + }); + t.deepEqual(await gzipOnly.select(), { + bundle: { kind: "combined", url: gzipOnly.assets[0].url }, + compressionMethod: "gzip", + }); + + for (const [platform, options] of [ + [BundlePlatform.Win64, {}], + [BundlePlatform.Linux64, { cliVersion: "2.18.4" }], + [BundlePlatform.Linux64, { cliVersion: undefined }], + [BundlePlatform.Linux64, { tarSupportsZstd: false }], + ] as const) { + const selection = await releaseFixture({ + assetNames: [ + `codeql-bundle-${platform}.tar.zst`, + `codeql-bundle-${platform}.tar.gz`, + ], + }).select({ platform, ...options }); + t.is( + selection.compressionMethod, + "gzip", + `${platform} ${JSON.stringify(options)}`, + ); + } +}); + +test("selectBundle requires a combined bundle that we can extract", async (t) => { + await t.throwsAsync(releaseFixture({ assetNames: [JAVA] }).select(), { + instanceOf: ConfigurationError, + message: `No compatible CodeQL bundle was found in release ${RELEASE_PAGE}. Expected codeql-bundle-linux64.tar.zst or codeql-bundle-linux64.tar.gz.`, + }); + await t.throwsAsync( + releaseFixture({ assetNames: [COMBINED] }).select({ + tarSupportsZstd: false, + }), + { instanceOf: ConfigurationError, message: /Expected [^ ]+\.tar\.gz\.$/ }, + ); +}); + +test("getRelease propagates API errors", async (t) => { + const fixture = releaseFixture({ status: 404 }); + t.like(await t.throwsAsync(fixture.select()), { status: 404 }); + t.deepEqual(fixture.requests, [fixture.releaseAPIURL]); +}); + +test("getPublicRelease constructs download URLs without looking up the release", async (t) => { + const fixture = releaseFixture(); + const release = getPublicRelease({ ...REFERENCE, tagName: "nightly/v1+2" }); + const baseURL = + "https://github.com/octo/tools/releases/download/nightly/v1%2B2"; + t.is( + release.url, + "https://github.com/octo/tools/releases/tag/nightly/v1%2B2", + ); + t.deepEqual( + await selectBundle(fixture.state, release, { + ...OPTIONS, + cliVersion: undefined, + isLatestNightly: true, + }), + { + bundle: { + kind: "per-language", + url: `${baseURL}/${JAVA}`, + language: BuiltInLanguage.java, + combinedBundleURL: `${baseURL}/${COMBINED}`, + }, + compressionMethod: "zstd", + }, + ); + t.deepEqual(fixture.requests, []); +}); diff --git a/src/codeql-release.ts b/src/codeql-release.ts new file mode 100644 index 0000000000..23f18503d3 --- /dev/null +++ b/src/codeql-release.ts @@ -0,0 +1,188 @@ +import * as semver from "semver"; + +import { ActionState } from "./action-common"; +import { CodeQLBundle, getCodeQLBundleName } from "./codeql-bundle"; +import { CODEQL_VERSION_ZSTD_BUNDLE } from "./feature-flags"; +import { + getPerLanguageBundleLanguage, + logMissingPerLanguageBundle, +} from "./per-language-bundles"; +import { BundlePlatform } from "./platform"; +import type { CompressionMethod } from "./tar"; +import { ConfigurationError, GitHubVariant } from "./util"; + +/** Identifies a release on a GitHub instance. */ +export interface CodeQLReleaseReference { + serverURL: string; + owner: string; + repo: string; + tagName: string; +} + +/** A GitHub release that contains CodeQL bundles. */ +export interface CodeQLRelease { + /** The release's web page, for messages. */ + url: string; + /** Returns the download URL for an asset, or `undefined` if the release doesn't have it. */ + getAssetURL(name: string): string | undefined; +} + +/** + * Encodes a tag for use in a URL path. Slashes stay as path separators, as in GitHub's release URLs + * for tags like `build/123`. + */ +function encodeTag(tagName: string): string { + return tagName.split("/").map(encodeURIComponent).join("/"); +} + +function getReleasePageURL(reference: CodeQLReleaseReference): string { + const { serverURL, owner, repo, tagName } = reference; + return `${serverURL}/${owner}/${repo}/releases/tag/${encodeTag(tagName)}`; +} + +/** + * Looks up a release on the current GitHub instance, which works for private repositories. + * + * The API client determines which instance we query, so `reference.serverURL` is only used for the + * release page URL in messages. The asset URLs are REST API endpoints, which accept the token and + * return the file when requested with `Accept: application/octet-stream`, unlike browser download + * URLs. + */ +export async function getRelease( + { apiClient }: ActionState<["Api"]>, + reference: CodeQLReleaseReference, +): Promise { + const { owner, repo, tagName } = reference; + const { data: release } = await apiClient.rest.repos.getReleaseByTag({ + owner, + repo, + tag: tagName, + }); + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => + release.assets.find((asset) => asset.name === name)?.url, + }; +} + +/** + * Refers to a public release without looking it up. Every asset gets a download URL, so a missing + * asset shows up as a failed download. + */ +export function getPublicRelease( + reference: CodeQLReleaseReference, +): CodeQLRelease { + const { serverURL, owner, repo, tagName } = reference; + return { + url: getReleasePageURL(reference), + getAssetURL: (name) => + `${serverURL}/${owner}/${repo}/releases/download/${encodeTag(tagName)}/${name}`, + }; +} + +/** Describes the job and runner that we are selecting a bundle for. */ +export interface BundleSelectionOptions { + /** Explicit `languages` input, which determines whether a per-language bundle is eligible. */ + rawLanguages: string[] | undefined; + /** The CLI version in the release, if known. */ + cliVersion: string | undefined; + platform: BundlePlatform | undefined; + variant: GitHubVariant; + tarSupportsZstd: boolean; + /** Whether the release is the latest nightly, whose CLI version we don't know yet. */ + isLatestNightly?: boolean; +} + +/** A bundle selected from a release. */ +export interface BundleSelection { + bundle: CodeQLBundle; + compressionMethod: CompressionMethod; + /** The release lacks the eligible per-language bundle, so we selected the combined bundle. */ + perLanguageBundleFallback?: true; +} + +/** Returns the compression methods that we can extract, most preferred first. */ +function getCompressionMethods({ + cliVersion, + isLatestNightly, + platform, + tarSupportsZstd, +}: BundleSelectionOptions): CompressionMethod[] { + if (!tarSupportsZstd) { + return ["gzip"]; + } + const preferZstd = + // In testing, gzip performs better than zstd on Windows. + platform !== BundlePlatform.Win64 && + // Standard bundles have zstd archives from this version, and so does the latest nightly. For a + // release we looked up, gzip comes next if it lacks the zstd archive. + (isLatestNightly || + (cliVersion !== undefined && + semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE))); + return preferZstd ? ["zstd", "gzip"] : ["gzip", "zstd"]; +} + +/** + * Selects a per-language bundle if the job is eligible for one, and otherwise the combined bundle. + * We only select a per-language bundle from a release that also has a combined bundle with the same + * compression, so that we can fall back to it. + * + * If we looked up the release, we can tell that an eligible per-language bundle is missing, so we + * warn and select the combined bundle straight away. A public release gives every asset a URL, so a + * missing per-language bundle only shows up as a 404 when downloading, which is when we fall back. + * + * Throws a `ConfigurationError` if the release has no combined bundle that we can extract. + */ +export async function selectBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, + release: CodeQLRelease, + options: BundleSelectionOptions, +): Promise { + const { logger } = action; + const compressionMethods = getCompressionMethods(options); + for (const compressionMethod of compressionMethods) { + const combinedBundleName = getCodeQLBundleName( + compressionMethod, + options.platform, + ); + const combinedBundleURL = release.getAssetURL(combinedBundleName); + if (combinedBundleURL === undefined) { + continue; + } + const combined: BundleSelection = { + bundle: { kind: "combined", url: combinedBundleURL }, + compressionMethod, + }; + + const language = await getPerLanguageBundleLanguage(action, { + ...options, + compressionMethod, + }); + if (language === undefined) { + logger.info( + `Selected CodeQL bundle ${combinedBundleName} from ${release.url}.`, + ); + return combined; + } + const name = getCodeQLBundleName( + compressionMethod, + options.platform, + language, + ); + const url = release.getAssetURL(name); + if (url === undefined) { + logMissingPerLanguageBundle(action, language, release.url); + return { ...combined, perLanguageBundleFallback: true }; + } + logger.info(`Selected CodeQL bundle ${name} from ${release.url}.`); + return { + bundle: { kind: "per-language", url, language, combinedBundleURL }, + compressionMethod, + }; + } + throw new ConfigurationError( + `No compatible CodeQL bundle was found in release ${release.url}. Expected ${compressionMethods + .map((method) => getCodeQLBundleName(method, options.platform)) + .join(" or ")}.`, + ); +} diff --git a/src/per-language-bundles.ts b/src/per-language-bundles.ts index e0db649c7a..ef64df265d 100644 --- a/src/per-language-bundles.ts +++ b/src/per-language-bundles.ts @@ -132,15 +132,15 @@ export async function getPerLanguageBundleLanguage( return language; } -/** Explains why an eligible per-language bundle is being replaced by a combined bundle. */ -export function logPerLanguageBundleFallback( +/** Logs that the selected per-language bundle is missing, so we're using the combined bundle. */ +export function logMissingPerLanguageBundle( { logger }: ActionState<["Logger"]>, language: BuiltInLanguage, location: string, ): void { logger.warning( - `No per-language CodeQL bundle for '${language}' was found at ${location}, so ` + - "falling back to the bundle that contains all languages. This analysis will still " + - "produce correct results, but will take longer to set up.", + `Expected a per-language CodeQL bundle for '${language}' at ${location}, but it wasn't ` + + "found, so using the bundle that contains all languages instead. This analysis will " + + "still produce correct results, but will take longer to set up.", ); } diff --git a/src/setup-codeql.test.ts b/src/setup-codeql.test.ts index c33ac0700a..d8d4cfe258 100644 --- a/src/setup-codeql.test.ts +++ b/src/setup-codeql.test.ts @@ -428,7 +428,7 @@ test.serial( false, // useOverlayAwareDefaultCliVersion SAMPLE_DOTCOM_API_DETAILS, GitHubVariant.DOTCOM, - false, + true, // tarSupportsZstd features, logger, ); @@ -498,7 +498,7 @@ test.serial( false, // useOverlayAwareDefaultCliVersion SAMPLE_DOTCOM_API_DETAILS, GitHubVariant.DOTCOM, - false, + true, // tarSupportsZstd features, logger, ); @@ -1214,22 +1214,14 @@ test.serial( }, ); -for (const fallback of [false, true]) { +for (const status of [200, 404]) { test.serial( - `setupCodeQLBundle retains the selected release identity for an opaque asset URL${fallback ? " with fallback" : ""}`, + `getCodeQLSource falls back to the public release when the Action's release ${status === 200 ? "lacks a compatible bundle" : "can't be found"}`, async (t) => { sinon.stub(process, "platform").value("linux"); sinon.stub(process, "arch").value("x64"); sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); - process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; - sinon.stub(tar, "isZstdAvailable").resolves({ - available: true, - foundZstdBinary: true, - }); const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName; - const assetURL = - "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123"; - const combinedURL = `${assetURL}4`; const fetchRelease = sinon .stub, ReturnType>() .callsFake( @@ -1237,23 +1229,94 @@ for (const fallback of [false, true]) { new Response( JSON.stringify({ assets: [ - { name: "codeql-bundle-java-linux64.tar.zst", url: assetURL }, { - name: "codeql-bundle-linux64.tar.zst", - url: combinedURL, + name: "codeql-bundle-java-linux64.tar.zst", + url: "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/1", }, ], }), - { headers: { "content-type": "application/json" } }, + { status, headers: { "content-type": "application/json" } }, ), ); const client = github.getOctokit("123", { request: { fetch: fetchRelease }, }); sinon.stub(api, "getApiClient").value(() => client); + + await withTmpDir(async (tmpDir) => { + setupActionsVars(tmpDir, tmpDir, { + GITHUB_ACTION_REPOSITORY: "codeql-testing/action-fork", + }); + const source = await setupCodeql.getCodeQLSource( + undefined, + PER_LANGUAGE_CLI_VERSION, + undefined, // rawLanguages + false, // useOverlayAwareDefaultCliVersion + SAMPLE_DOTCOM_API_DETAILS, + GitHubVariant.DOTCOM, + true, // tarSupportsZstd + createFeatures([]), + getRunnerLogger(true), + ); + + t.true(fetchRelease.calledOnce); + t.like(source, { + sourceType: "download", + bundle: { + kind: "combined", + url: `https://github.com/github/codeql-action/releases/download/${tag}/codeql-bundle-linux64.tar.zst`, + }, + }); + }); + }, + ); +} + +for (const scenario of ["per-language", "fallback", "missing"] as const) { + const suffix = { + "per-language": "", + fallback: " with fallback", + missing: " when the release lacks the per-language bundle", + }[scenario]; + test.serial( + `setupCodeQLBundle retains the selected release identity for an opaque asset URL${suffix}`, + async (t) => { + sinon.stub(process, "platform").value("linux"); + sinon.stub(process, "arch").value("x64"); + sinon.stub(actionsUtil, "isRunningLocalAction").returns(false); + process.env[ActionsEnvVars.RUNNER_ENVIRONMENT] = "github-hosted"; + sinon.stub(tar, "isZstdAvailable").resolves({ + available: true, + foundZstdBinary: true, + }); + const tag = PER_LANGUAGE_CLI_VERSION.enabledVersions[0].tagName; + const assetURL = + "https://api.github.com/repos/codeql-testing/action-fork/releases/assets/123"; + const combinedURL = `${assetURL}4`; + const assets = [ + { name: "codeql-bundle-linux64.tar.zst", url: combinedURL }, + ]; + if (scenario !== "missing") { + assets.push({ + name: "codeql-bundle-java-linux64.tar.zst", + url: assetURL, + }); + } + const fetchRelease = sinon + .stub, ReturnType>() + .callsFake( + async () => + new Response(JSON.stringify({ assets }), { + headers: { "content-type": "application/json" }, + }), + ); + const client = github.getOctokit("123", { + request: { fetch: fetchRelease }, + }); + sinon.stub(api, "getApiClient").value(() => client); const authorizationSpy = sinon.spy(api, "getAuthorizationHeaderFor"); const extractStub = stubDownloadAndExtract(); - if (fallback) { + if (scenario === "fallback") { extractStub.onFirstCall().rejects(new HTTPError("Not Found", 404)); } @@ -1273,41 +1336,46 @@ for (const fallback of [false, true]) { getRunnerLogger(true), ); - t.true(fetchRelease.calledTwice); + const usesPerLanguageBundle = scenario === "per-language"; + t.true(fetchRelease.calledOnce); t.is( fetchRelease.firstCall.args[0], `https://api.github.com/repos/codeql-testing/action-fork/releases/tags/${tag}`, ); - t.is(extractStub.callCount, fallback ? 2 : 1); - t.is(extractStub.firstCall.args[0], assetURL); - t.is(extractStub.lastCall.args[0], fallback ? combinedURL : assetURL); - t.is(authorizationSpy.callCount, extractStub.callCount); - t.is(authorizationSpy.firstCall.args[2], assetURL); + t.is(extractStub.callCount, scenario === "fallback" ? 2 : 1); t.is( - authorizationSpy.lastCall.args[2], - fallback ? combinedURL : assetURL, + extractStub.firstCall.args[0], + scenario === "missing" ? combinedURL : assetURL, + ); + t.is( + extractStub.lastCall.args[0], + usesPerLanguageBundle ? assetURL : combinedURL, + ); + t.deepEqual( + authorizationSpy.getCalls().map((call) => call.args[2]), + extractStub.getCalls().map((call) => call.args[0]), ); t.is(extractStub.lastCall.args[3], "token token"); t.is(result.toolsVersion, MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION); t.is( result.toolsDownloadStatusReport?.perLanguage?.tools_bundle_language, - fallback ? undefined : BuiltInLanguage.java, + usesPerLanguageBundle ? BuiltInLanguage.java : undefined, ); t.is( result.toolsDownloadStatusReport?.perLanguage ?.tools_per_language_bundle_fallback, - fallback ? true : undefined, + usesPerLanguageBundle ? undefined : true, ); - if (fallback) { + if (usesPerLanguageBundle) { + t.is(path.dirname(result.codeqlFolder), tmpDir); + t.deepEqual(toolcache.findAllVersions("CodeQL"), []); + t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); + } else { t.is( result.codeqlFolder, toolcache.find("CodeQL", MIN_PER_LANGUAGE_BUNDLE_CLI_VERSION), ); t.true(fs.existsSync(`${result.codeqlFolder}.complete`)); - } else { - t.is(path.dirname(result.codeqlFolder), tmpDir); - t.deepEqual(toolcache.findAllVersions("CodeQL"), []); - t.false(fs.existsSync(`${result.codeqlFolder}.complete`)); } }); }, @@ -1420,9 +1488,10 @@ for (const bundle of ["per-language", "combined", "fallback"] as const) { t.deepEqual(downloadSpy.secondCall.args[0], { ...source, bundle: { kind: "combined", url: combinedURL }, + perLanguageBundleFallback: true, }); checkExpectedLogMessages(t, loggedMessages, [ - `No per-language CodeQL bundle for 'javascript' was found at ${perLanguageURL}`, + `Expected a per-language CodeQL bundle for 'javascript' at ${perLanguageURL}`, ]); } if (bundle === "per-language") { diff --git a/src/setup-codeql.ts b/src/setup-codeql.ts index d306d40d63..98767143a4 100644 --- a/src/setup-codeql.ts +++ b/src/setup-codeql.ts @@ -21,8 +21,14 @@ import { CodeQLBundle, CodeQLDownloadSource, getCodeQLBundleFromUrl, - getCodeQLBundleName, } from "./codeql-bundle"; +import { + BundleSelection, + BundleSelectionOptions, + getPublicRelease, + getRelease, + selectBundle, +} from "./codeql-release"; import * as defaults from "./defaults.json"; import { addNoLanguageDiagnostic, @@ -31,19 +37,14 @@ import { } from "./diagnostics"; import { EnvVar, getEnv } from "./environment"; import { - CODEQL_VERSION_ZSTD_BUNDLE, CodeQLDefaultVersionInfo, CodeQLVersionInfo, Feature, FeatureEnablement, } from "./feature-flags"; -import { BuiltInLanguage } from "./languages"; import { Logger } from "./logging"; import { getCodeQlVersionsForOverlayBaseDatabases } from "./overlay/caching"; -import { - getPerLanguageBundleLanguage, - logPerLanguageBundleFallback, -} from "./per-language-bundles"; +import { logMissingPerLanguageBundle } from "./per-language-bundles"; import { getBundlePlatform } from "./platform"; import * as tar from "./tar"; import { @@ -88,12 +89,19 @@ export function getCodeQLActionRepository(logger: Logger): string { return util.getRequiredEnvParam("GITHUB_ACTION_REPOSITORY"); } -async function getCodeQLBundleDownloadURL( +/** + * Selects a bundle from the first release tagged `tagName` that has a compatible bundle, trying the + * Action repositories on this GitHub instance before the canonical Action on GitHub.com. If we + * can't look up a release, or it has no compatible bundle, we move on to the next repository. We + * assume that the public release on GitHub.com has every bundle. + */ +async function selectDefaultBundle( + action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, tagName: string, apiDetails: api.GitHubApiDetails, - codeQLBundleName: string, - logger: Logger, -): Promise { + options: BundleSelectionOptions, +): Promise { + const { logger } = action; const codeQLActionRepository = getCodeQLActionRepository(logger); const potentialDownloadSources = [ // This GitHub instance, and this Action. @@ -110,37 +118,38 @@ async function getCodeQLBundleDownloadURL( return !self.slice(0, index).some((other) => deepEqual(source, other)); }, ); - for (const downloadSource of uniqueDownloadSources) { - const [apiURL, repository] = downloadSource; + for (const [serverURL, repository] of uniqueDownloadSources) { // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public. if ( - apiURL === util.GITHUB_DOTCOM_URL && + serverURL === util.GITHUB_DOTCOM_URL && repository === CODEQL_DEFAULT_ACTION_REPOSITORY ) { break; } - const [repositoryOwner, repositoryName] = repository.split("/"); + const [owner, repo] = repository.split("/"); try { - const release = await api.getApiClient().rest.repos.getReleaseByTag({ - owner: repositoryOwner, - repo: repositoryName, - tag: tagName, - }); - for (const asset of release.data.assets) { - if (asset.name === codeQLBundleName) { - logger.info( - `Found CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} with URL ${asset.url}.`, - ); - return asset.url; - } - } + const release = await getRelease( + { apiClient: api.getApiClient() }, + { serverURL, owner, repo, tagName }, + ); + return await selectBundle(action, release, options); } catch (e) { logger.info( - `Looked for CodeQL bundle ${codeQLBundleName} in ${repository} on ${apiURL} but got error ${e}.`, + `Looked for CodeQL bundles in release ${tagName} of ${repository} on ${serverURL} but got error ${e}.`, ); } } - return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; + const [owner, repo] = CODEQL_DEFAULT_ACTION_REPOSITORY.split("/"); + return selectBundle( + action, + getPublicRelease({ + serverURL: util.GITHUB_DOTCOM_URL, + owner, + repo, + tagName, + }), + options, + ); } function tryGetBundleVersionFromTagName( @@ -374,6 +383,20 @@ async function resolveDefaultCliVersion( * Determines where the CodeQL CLI we want to use comes from. This can be from a local file, * the Actions toolcache, or a download. * + * We handle the `tools` input in this order: + * + * - A local path is extracted without using the toolcache. + * - `nightly` or `nightly-latest`, or the `force_nightly` feature flag in a dynamic workflow, + * selects a bundle from the latest nightly release. We then continue with that bundle's URL. + * - `linked`, or its old name `latest`, selects the version shipped with the Action. + * - `toolcache` selects the latest version in the toolcache, falling back to the default version + * outside dynamic workflows or if there isn't one. + * - Any other value is the URL of a bundle. + * - Without a `tools` input, we use the default version. + * + * Apart from a local path, we look for the resolved version in the toolcache before downloading. A + * cached version takes precedence even if the job could use a per-language bundle. + * * @param toolsInput The argument provided for the `tools` input, if any. * @param defaultCliVersion The default CLI version that's linked to the CodeQL Action. * @param rawLanguages Raw set of languages. @@ -479,6 +502,7 @@ export async function getCodeQLSource( { env: getEnv(), features, logger }, rawLanguages, variant, + tarSupportsZstd, ); toolsInput = bundle.url; } @@ -556,7 +580,9 @@ export async function getCodeQLSource( tagName = version.tagName; } } else if (toolsInput !== undefined) { - // If a tools URL was provided, then use that. + // Any other value is a bundle URL, including one we selected from the latest nightly above. + // We use the version in its tag, if any, for the toolcache, so we assume that bundles with the + // same version are the same build, whichever repository they're in. tagName = tryGetTagNameFromUrl(toolsInput, logger); url = toolsInput; @@ -701,58 +727,28 @@ export async function getCodeQLSource( } let compressionMethod: tar.CompressionMethod; + let perLanguageBundleFallback: true | undefined; if (!url) { - const bundleTagName = tagName; - if (bundleTagName === undefined) { + if (tagName === undefined) { throw new Error( "Could not determine a release tag for the requested CodeQL bundle.", ); } - - compressionMethod = - cliVersion !== undefined && - (await useZstdBundle(cliVersion, tarSupportsZstd)) - ? "zstd" - : "gzip"; - - const platform = getBundlePlatform(); - const perLanguageBundleLanguage = await getPerLanguageBundleLanguage( - { env: getEnv(), features, logger }, - { - rawLanguages, - cliVersion, - compressionMethod, - platform, - variant, - }, - ); - - // Resolves the combined or per-language bundle URL for the requested release. - const resolveBundleURL = (language?: BuiltInLanguage) => - getCodeQLBundleDownloadURL( - bundleTagName, + ({ bundle, compressionMethod, perLanguageBundleFallback } = + await selectDefaultBundle( + { env: getEnv(), features, logger }, + tagName, apiDetails, - getCodeQLBundleName(compressionMethod, platform, language), - logger, - ); - - const combinedBundleURL = await resolveBundleURL(); - if (perLanguageBundleLanguage !== undefined) { - logger.info( - `Selected the per-language CodeQL bundle for '${perLanguageBundleLanguage}'.`, - ); - url = await resolveBundleURL(perLanguageBundleLanguage); - bundle = { - kind: "per-language", - url, - language: perLanguageBundleLanguage, - combinedBundleURL, - }; - } else { - url = combinedBundleURL; - bundle = { kind: "combined", url }; - } + { + rawLanguages, + cliVersion, + platform: getBundlePlatform(), + variant, + tarSupportsZstd, + }, + )); + url = bundle.url; } else { const method = tar.inferCompressionMethod(url); if (method === undefined) { @@ -763,6 +759,8 @@ export async function getCodeQLSource( } compressionMethod = method; + // Keep the bundle we selected from the latest nightly, which records the combined bundle to + // fall back to. Otherwise, classify the explicit URL, which gets no fallback. bundle ??= getCodeQLBundleFromUrl(url); if (bundle.kind === "per-language") { logger.info( @@ -781,6 +779,7 @@ export async function getCodeQLSource( bundleVersion, cliVersion, compressionMethod, + ...(perLanguageBundleFallback ? { perLanguageBundleFallback } : {}), sourceType: "download", toolsVersion: resolvedVersion ?? "unknown", }; @@ -868,7 +867,12 @@ export const downloadCodeQL = async function ( ...statusReport, perLanguage: { tools_bundle_language: bundle.language }, } - : statusReport, + : source.perLanguageBundleFallback + ? { + ...statusReport, + perLanguage: { tools_per_language_bundle_fallback: true }, + } + : statusReport, }; }; @@ -1113,12 +1117,13 @@ export async function downloadCodeQLBundle( ) { throw e; } - logPerLanguageBundleFallback(action, bundle.language, bundle.url); + logMissingPerLanguageBundle(action, bundle.language, bundle.url); const result = await downloadCodeQL( { ...source, bundle: { kind: "combined", url: bundle.combinedBundleURL }, + perLanguageBundleFallback: true, }, apiDetails, tarVersion, @@ -1130,24 +1135,11 @@ export async function downloadCodeQLBundle( statusReport: { ...result.statusReport, totalDurationMs: util.durationMsSince(startTime), - perLanguage: { tools_per_language_bundle_fallback: true }, }, }; } } -async function useZstdBundle( - cliVersion: string, - tarSupportsZstd: boolean, -): Promise { - return ( - // In testing, gzip performs better than zstd on Windows. - process.platform !== "win32" && - tarSupportsZstd && - semver.gte(cliVersion, CODEQL_VERSION_ZSTD_BUNDLE) - ); -} - function getTempExtractionDir(tempDir: string) { return path.join(tempDir, uuidV4()); } @@ -1160,27 +1152,9 @@ async function getLatestNightlyBundle( action: ActionState<["Logger", "ReadOnlyEnv", "FeatureFlags"]>, rawLanguages: string[] | undefined, variant: util.GitHubVariant, + tarSupportsZstd: boolean, ): Promise { - const { logger } = action; - const zstdAvailability = await tar.isZstdAvailable(logger); - // The nightly is guaranteed to have a zstd bundle - const compressionMethod = (await useZstdBundle( - CODEQL_VERSION_ZSTD_BUNDLE, - zstdAvailability.available, - )) - ? "zstd" - : "gzip"; - - const platform = getBundlePlatform(); - const language = await getPerLanguageBundleLanguage(action, { - rawLanguages, - cliVersion: undefined, - compressionMethod, - platform, - variant, - isLatestNightly: true, - }); - + let tagName: string; try { // Since nightlies are prereleases, we can't just download the latest release // on the repository. So instead we need to find the latest pre-release @@ -1196,26 +1170,31 @@ async function getLatestNightlyBundle( if (!latestRelease) { throw new Error("Could not find the latest nightly release."); } - const assetUrl = (name: string) => - `https://github.com/${CODEQL_NIGHTLIES_REPOSITORY_OWNER}/${CODEQL_NIGHTLIES_REPOSITORY_NAME}/releases/download/${latestRelease.tag_name}/${name}`; - const url = assetUrl( - getCodeQLBundleName(compressionMethod, platform, language), - ); - return language === undefined - ? { kind: "combined", url } - : { - kind: "per-language", - url, - language, - combinedBundleURL: assetUrl( - getCodeQLBundleName(compressionMethod, platform), - ), - }; + tagName = latestRelease.tag_name; } catch (e) { throw new Error( `Failed to retrieve the latest nightly release: ${util.wrapError(e)}`, ); } + + const { bundle } = await selectBundle( + action, + getPublicRelease({ + serverURL: util.GITHUB_DOTCOM_URL, + owner: CODEQL_NIGHTLIES_REPOSITORY_OWNER, + repo: CODEQL_NIGHTLIES_REPOSITORY_NAME, + tagName, + }), + { + rawLanguages, + cliVersion: undefined, + platform: getBundlePlatform(), + variant, + tarSupportsZstd, + isLatestNightly: true, + }, + ); + return bundle; } /**