Skip to content

Commit c2fd8f5

Browse files
authored
Merge pull request #4081 from github/mario-campos/version-cache-to-disk
Persist CodeQL version output to file rather than environment
2 parents 5008eff + c56f48e commit c2fd8f5

11 files changed

Lines changed: 1230 additions & 1070 deletions

lib/entry-points.js

Lines changed: 921 additions & 902 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/cli/output-cache.test.ts

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
import * as fs from "fs";
2+
import path from "path";
3+
4+
import test from "ava";
5+
6+
import { EnvVar } from "../environment";
7+
import { getRunnerLogger } from "../logging";
8+
import { getTestEnv, setupTests } from "../testing-utils";
9+
import * as util from "../util";
10+
11+
import * as outputCache from "./output-cache";
12+
13+
setupTests(test);
14+
15+
const logger = getRunnerLogger(true);
16+
17+
test.serial(
18+
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
19+
async (t) => {
20+
await util.withTmpDir(async (tmpDir: string) => {
21+
const cacheFile = path.join(tmpDir, "codeql-action-command-cache.json");
22+
fs.writeFileSync(
23+
cacheFile,
24+
JSON.stringify({
25+
cmd: "/path/to/codeql",
26+
entries: { version: { version: "2.20.0" } },
27+
}),
28+
"utf8",
29+
);
30+
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
31+
t.deepEqual(
32+
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
33+
{
34+
version: "2.20.0",
35+
},
36+
);
37+
});
38+
},
39+
);
40+
41+
test.serial(
42+
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
43+
async (t) => {
44+
await util.withTmpDir(async (tmpDir: string) => {
45+
const cacheFile = path.join(tmpDir, "version.json");
46+
fs.writeFileSync(
47+
cacheFile,
48+
JSON.stringify({
49+
cmd: "/path/to/other-codeql",
50+
version: { version: "2.20.0" },
51+
}),
52+
"utf8",
53+
);
54+
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
55+
t.is(
56+
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
57+
undefined,
58+
);
59+
});
60+
},
61+
);
62+
63+
test.serial(
64+
"getCachedCodeQlVersion ignores a malformed persisted value",
65+
async (t) => {
66+
await util.withTmpDir(async (tmpDir: string) => {
67+
const cacheFile = path.join(tmpDir, "version.json");
68+
fs.writeFileSync(cacheFile, "not valid json", "utf8");
69+
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
70+
t.is(
71+
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
72+
undefined,
73+
);
74+
});
75+
},
76+
);
77+
78+
test.serial(
79+
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
80+
async (t) => {
81+
await util.withTmpDir(async (tmpDir: string) => {
82+
const cacheFile = path.join(tmpDir, "version.json");
83+
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
84+
85+
const testValues = [
86+
{ cmd: "/path/to/codeql" },
87+
{ entries: { version: { version: "2.20.0" } } },
88+
{ cmd: "/path/to/codeql", entries: {} },
89+
{ cmd: "/path/to/codeql", entries: null },
90+
{ cmd: "/path/to/codeql", entries: { version: {} } },
91+
{ cmd: "/path/to/codeql", entries: { version: null } },
92+
{ cmd: "/path/to/codeql", entries: { version: "2.20.0" } },
93+
{ cmd: "/path/to/codeql", entries: { version: { version: null } } },
94+
{ cmd: "/path/to/codeql", entries: { version: { version: 2.2 } } },
95+
{ cmd: "/path/to/codeql", entries: { version: { version: 2 } } },
96+
{
97+
cmd: "/path/to/codeql",
98+
entries: { version: { version: "2.20.0", overlayVersion: "1" } },
99+
},
100+
{
101+
cmd: "/path/to/codeql",
102+
entries: { version: { version: "2.20.0", features: "nope" } },
103+
},
104+
].map((v) => JSON.stringify(v));
105+
106+
for (const value of testValues) {
107+
fs.writeFileSync(cacheFile, value, "utf8");
108+
t.is(
109+
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
110+
undefined,
111+
value,
112+
);
113+
}
114+
});
115+
},
116+
);
117+
118+
test.serial("getCachedCodeQlVersion ignores non-existent file", async (t) => {
119+
await util.withTmpDir(async (tmpDir: string) => {
120+
const env = getTestEnv({ [EnvVar.TEMP]: tmpDir });
121+
t.notThrows(() => {
122+
t.is(
123+
outputCache.getCachedCodeQlVersion(logger, env, "/path/to/codeql"),
124+
undefined,
125+
);
126+
});
127+
});
128+
});

src/cli/output-cache.ts

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
import * as fs from "fs";
2+
import path from "path";
3+
4+
import { getTemporaryDirectory } from "../actions-util";
5+
import { Env } from "../environment";
6+
import { Logger } from "../logging";
7+
8+
import type { VersionInfo } from "./types";
9+
10+
/**
11+
* The keys of the command cache. Each key corresponds to a command whose output we cache.
12+
*/
13+
export type CommandCacheKey = string;
14+
15+
/**
16+
* The type of the command cache that is persisted to disk.
17+
*/
18+
export interface OutputCache {
19+
cmd: string;
20+
entries: Record<CommandCacheKey, unknown>;
21+
}
22+
23+
/**
24+
* The name of the temporary file that backs the on-disk cache of
25+
* CLI responses between workflow steps.
26+
*/
27+
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
28+
29+
/**
30+
* The module-global variable that caches the CodeQL CLI version in-memory.
31+
*/
32+
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
33+
34+
/**
35+
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
36+
* which exercise multiple "steps" within a single process.
37+
*/
38+
export function resetCachedCodeQlVersion(): void {
39+
cachedCodeQlVersion = undefined;
40+
}
41+
42+
/**
43+
* Returns the path to the temporary file that backs the
44+
* on-disk cache of CLI responses between workflow steps.
45+
*/
46+
function getCommandCacheFilePath(env: Env): string {
47+
return path.join(getTemporaryDirectory(env), COMMAND_CACHE_FILENAME);
48+
}
49+
50+
/**
51+
* Caches the CodeQL CLI version both in-memory and on disk.
52+
* @param env The environment variables to use.
53+
* @param cmd The path to the CodeQL CLI.
54+
* @param version The version information to cache.
55+
*/
56+
export function cacheCodeQlVersion(
57+
env: Env,
58+
cmd: string,
59+
version: VersionInfo,
60+
): void {
61+
if (cachedCodeQlVersion !== undefined) {
62+
throw new Error("cacheCodeQlVersion() should be called only once");
63+
}
64+
cachedCodeQlVersion = version;
65+
const outputCache = {
66+
cmd,
67+
entries: { version },
68+
} satisfies OutputCache;
69+
// Persist the version so that subsequent Actions steps, which run in separate
70+
// processes, can reuse it rather than invoking `codeql version` again. We
71+
// record the CLI path so that a different step using a different CodeQL bundle
72+
// doesn't pick up a stale version.
73+
fs.writeFileSync(
74+
getCommandCacheFilePath(env),
75+
JSON.stringify(outputCache),
76+
"utf8",
77+
);
78+
}
79+
80+
/**
81+
* Returns the cached CodeQL CLI version, if any.
82+
* @param logger The logger to use for logging messages.
83+
* @param env The environment variables to use.
84+
* @param cmd The path to the CodeQL CLI.
85+
*/
86+
export function getCachedCodeQlVersion(
87+
logger: Logger,
88+
env: Env,
89+
cmd?: string,
90+
): undefined | VersionInfo {
91+
if (cachedCodeQlVersion !== undefined) {
92+
return cachedCodeQlVersion;
93+
}
94+
// Fall back to the value persisted by an earlier Actions step, if any. This is
95+
// best-effort: any malformed or mismatched value is ignored so that the caller
96+
// invokes `codeql version` instead.
97+
let serialized: string;
98+
try {
99+
serialized = fs.readFileSync(getCommandCacheFilePath(env), "utf8");
100+
} catch (e) {
101+
logger.debug(
102+
`Cannot read CLI-cache file ${getCommandCacheFilePath(env)}: ${e}`,
103+
);
104+
return undefined;
105+
}
106+
let persisted: unknown;
107+
try {
108+
persisted = JSON.parse(serialized);
109+
} catch (e) {
110+
logger.debug(`Cannot parse CLI-cache data as JSON: ${e}`);
111+
return undefined;
112+
}
113+
if (
114+
!isOutputCache(persisted) ||
115+
(cmd !== undefined && persisted.cmd !== cmd)
116+
) {
117+
return undefined;
118+
}
119+
// Memoize the parsed value so that subsequent calls in this process don't
120+
// re-parse the environment variable.
121+
cachedCodeQlVersion = persisted.entries.version as VersionInfo;
122+
return cachedCodeQlVersion;
123+
}
124+
125+
/**
126+
* Determines whether a value is a `VersionInfo` object.
127+
* @param x The value to test
128+
*/
129+
function isVersionInfo(x: unknown): x is VersionInfo {
130+
const candidate = x as Partial<VersionInfo> | null;
131+
return (
132+
typeof candidate === "object" &&
133+
candidate !== null &&
134+
typeof candidate.version === "string" &&
135+
(candidate.features === undefined ||
136+
(typeof candidate.features === "object" &&
137+
candidate.features !== null)) &&
138+
(candidate.overlayVersion === undefined ||
139+
typeof candidate.overlayVersion === "number")
140+
);
141+
}
142+
143+
/**
144+
* Determines whether a value is a `OutputCache` object.
145+
* @param x The value to test
146+
*/
147+
function isOutputCache(x: unknown): x is OutputCache {
148+
const candidate = x as Partial<OutputCache> | null;
149+
return (
150+
typeof candidate === "object" &&
151+
candidate !== null &&
152+
typeof candidate.cmd === "string" &&
153+
candidate.entries !== undefined &&
154+
isVersionInfo(candidate.entries.version)
155+
);
156+
}

src/cli/types.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
export interface VersionInfo {
2+
version: string;
3+
features?: { [name: string]: boolean };
4+
/**
5+
* The overlay version helps deal with backward incompatible changes for
6+
* overlay analysis. When a precompiled query pack reports the same overlay
7+
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
8+
* analysis with that pack. Otherwise, if the overlay versions are different,
9+
* or if either the pack or the CLI does not report an overlay version,
10+
* we need to revert to non-overlay analysis.
11+
*/
12+
overlayVersion?: number;
13+
}

src/codeql.ts

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@ import {
1212
runTool,
1313
} from "./actions-util";
1414
import * as api from "./api-client";
15+
import * as outputCache from "./cli/output-cache";
16+
import type { VersionInfo } from "./cli/types";
1517
import { CliError, wrapCliConfigurationError } from "./cli-errors";
1618
import { appendExtraQueryExclusions, type Config } from "./config-utils";
1719
import { DocUrl } from "./doc-url";
18-
import { EnvVar } from "./environment";
20+
import { EnvVar, getEnv } from "./environment";
1921
import {
2022
CodeQLDefaultVersionInfo,
2123
Feature,
@@ -214,20 +216,6 @@ export interface CodeQL {
214216
): Promise<void>;
215217
}
216218

217-
export interface VersionInfo {
218-
version: string;
219-
features?: { [name: string]: boolean };
220-
/**
221-
* The overlay version helps deal with backward incompatible changes for
222-
* overlay analysis. When a precompiled query pack reports the same overlay
223-
* version as the CodeQL CLI, we can use the CodeQL CLI to perform overlay
224-
* analysis with that pack. Otherwise, if the overlay versions are different,
225-
* or if either the pack or the CLI does not report an overlay version,
226-
* we need to revert to non-overlay analysis.
227-
*/
228-
overlayVersion?: number;
229-
}
230-
231219
export interface ResolveDatabaseOutput {
232220
overlayBaseSpecifier?: string;
233221
}
@@ -503,7 +491,7 @@ async function getCodeQLForCmd(
503491
return cmd;
504492
},
505493
async getVersion() {
506-
let result = util.getCachedCodeQlVersion(cmd);
494+
let result = outputCache.getCachedCodeQlVersion(logger, getEnv(), cmd);
507495
if (result === undefined) {
508496
result = await runCliJson<VersionInfo>(
509497
cmd,
@@ -512,7 +500,7 @@ async function getCodeQLForCmd(
512500
noStreamStdout: true,
513501
},
514502
);
515-
util.cacheCodeQlVersion(cmd, result);
503+
outputCache.cacheCodeQlVersion(getEnv(), cmd, result);
516504
}
517505
return result;
518506
},

src/environment.ts

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,6 @@ export enum EnvVar {
3939
*/
4040
CODE_SCANNING_REF = "CODE_SCANNING_REF",
4141

42-
/**
43-
* `PersistedVersionInfo` for the CodeQL CLI, so later Actions steps can reuse it instead of
44-
* invoking `codeql version` again.
45-
*/
46-
CODEQL_VERSION_INFO = "CODEQL_ACTION_CLI_VERSION_INFO",
47-
4842
/** Whether the CodeQL Action has invoked the Go autobuilder. */
4943
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
5044

src/status-report.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
isSelfHostedRunner,
1515
} from "./actions-util";
1616
import { getAnalysisKey, getApiClient } from "./api-client";
17+
import { getCachedCodeQlVersion } from "./cli/output-cache";
1718
import type { Config } from "./config/action-config";
1819
import type { ComputedInput, InputName } from "./config/inputs";
1920
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
@@ -30,7 +31,6 @@ import { registryBaseSchema } from "./start-proxy/types";
3031
import {
3132
ConfigurationError,
3233
getRequiredEnvParam,
33-
getCachedCodeQlVersion,
3434
isInTestMode,
3535
GITHUB_DOTCOM_URL,
3636
DiskUsage,
@@ -376,7 +376,7 @@ export async function createStatusReportBase(
376376
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
377377
}
378378
const runnerOs = getRequiredEnvParam("RUNNER_OS");
379-
const codeQlCliVersion = getCachedCodeQlVersion();
379+
const codeQlCliVersion = getCachedCodeQlVersion(logger, getEnv());
380380
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
381381
const testingEnvironment = getTestingEnvironment();
382382
// re-export the testing environment variable so that it is available to subsequent steps,

0 commit comments

Comments
 (0)