From af9a3e7c5c9b6775ec25f0cc8f4d83c58c323ca3 Mon Sep 17 00:00:00 2001 From: Bailey Pearson Date: Thu, 6 Feb 2025 11:58:26 -0700 Subject: [PATCH] chore(NODE-6726): make install libmongocrypt script faster (#66) --- .github/docker/Dockerfile.glibc | 2 +- .github/scripts/get-commit-from-ref.sh | 14 ++++ .github/scripts/libmongocrypt.mjs | 100 ++++--------------------- .github/scripts/utils.mjs | 89 ++++++++++++++++++++++ 4 files changed, 120 insertions(+), 85 deletions(-) create mode 100644 .github/scripts/get-commit-from-ref.sh create mode 100644 .github/scripts/utils.mjs diff --git a/.github/docker/Dockerfile.glibc b/.github/docker/Dockerfile.glibc index b22a180..cfd02fc 100644 --- a/.github/docker/Dockerfile.glibc +++ b/.github/docker/Dockerfile.glibc @@ -11,7 +11,7 @@ ENV PATH=$PATH:/nodejs/bin WORKDIR /mongodb-client-encryption COPY . . -RUN apt-get -qq update && apt-get -qq install -y python3 build-essential && ldd --version +RUN apt-get -qq update && apt-get -qq install -y python3 build-essential git && ldd --version RUN npm run install:libmongocrypt diff --git a/.github/scripts/get-commit-from-ref.sh b/.github/scripts/get-commit-from-ref.sh new file mode 100644 index 0000000..823b4aa --- /dev/null +++ b/.github/scripts/get-commit-from-ref.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env bash +set -o errexit + +git clone https://github.com/mongodb/libmongocrypt.git _libmongocrypt +cd _libmongocrypt + +git checkout --detach $REF + +COMMIT_HASH=$(git rev-parse HEAD) + +echo "COMMIT_HASH=$COMMIT_HASH" + +cd - +rm -rf _libmongocrypt diff --git a/.github/scripts/libmongocrypt.mjs b/.github/scripts/libmongocrypt.mjs index 0f83095..d61a7bd 100644 --- a/.github/scripts/libmongocrypt.mjs +++ b/.github/scripts/libmongocrypt.mjs @@ -8,14 +8,7 @@ import events from 'node:events'; import path from 'node:path'; import https from 'node:https'; import stream from 'node:stream/promises'; -import url from 'node:url'; - -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -/** Resolves to the root of this repository */ -function resolveRoot(...paths) { - return path.resolve(__dirname, '..', '..', ...paths); -} +import { buildLibmongocryptDownloadUrl, getLibmongocryptPrebuildName, resolveRoot, run } from './utils.mjs'; async function parseArguments() { const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8')); @@ -26,7 +19,6 @@ async function parseArguments() { clean: { short: 'c', type: 'boolean', default: false }, build: { short: 'b', type: 'boolean', default: false }, dynamic: { type: 'boolean', default: false }, - fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient 'skip-bindings': { type: 'boolean', default: false }, help: { short: 'h', type: 'boolean', default: false } }; @@ -46,7 +38,6 @@ async function parseArguments() { return { url: args.values.gitURL, ref: args.values.libVersion, - fastDownload: args.values.fastDownload, clean: args.values.clean, build: args.values.build, dynamic: args.values.dynamic, @@ -55,26 +46,6 @@ async function parseArguments() { }; } -/** `xtrace` style command runner, uses spawn so that stdio is inherited */ -async function run(command, args = [], options = {}) { - const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`; - console.error(commandDetails); - const proc = child_process.spawn(command, args, { - shell: process.platform === 'win32', - stdio: 'inherit', - cwd: resolveRoot('.'), - ...options - }); - await events.once(proc, 'exit'); - - if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); -} - -/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */ -function toFlags(object) { - return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`); -} - export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) { console.error('fetching libmongocrypt...', { url, ref }); await fs.rm(libmongocryptRoot, { recursive: true, force: true }); @@ -87,6 +58,11 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) { } export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) { + /** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */ + function toCLIFlags(object) { + return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`); + } + console.error('building libmongocrypt...'); const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build'); @@ -94,7 +70,7 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option await fs.rm(nodeBuildRoot, { recursive: true, force: true }); await fs.mkdir(nodeBuildRoot, { recursive: true }); - const CMAKE_FLAGS = toFlags({ + const CMAKE_FLAGS = toCLIFlags({ /** * We provide crypto hooks from Node.js binding to openssl (so disable system crypto) * TODO: NODE-5455 @@ -127,12 +103,12 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option const WINDOWS_CMAKE_FLAGS = process.platform === 'win32' // Windows is still called "win32" when it is 64-bit - ? toFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' }) + ? toCLIFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' }) : []; const DARWIN_CMAKE_FLAGS = process.platform === 'darwin' // The minimum darwin target version we want for - ? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' }) + ? toCLIFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' }) : []; const cmakeProgram = process.platform === 'win32' ? 'cmake.exe' : 'cmake'; @@ -149,11 +125,10 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option }); } -export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) { - const downloadURL = - ref === 'latest' - ? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz' - : `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`; +export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) { + const prebuild = getLibmongocryptPrebuildName(); + + const downloadURL = buildLibmongocryptDownloadUrl(ref, prebuild); console.error('downloading libmongocrypt...', downloadURL); const destination = resolveRoot(`_libmongocrypt-${ref}`); @@ -161,23 +136,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) await fs.rm(destination, { recursive: true, force: true }); await fs.mkdir(destination); - const platformMatrix = { - ['darwin-arm64']: 'macos', - ['darwin-x64']: 'macos', - ['linux-ppc64']: 'rhel-71-ppc64el', - ['linux-s390x']: 'rhel72-zseries-test', - ['linux-arm64']: 'ubuntu1804-arm64', - ['linux-x64']: 'rhel-70-64-bit', - ['win32-x64']: 'windows-test' - }; - - const detectedPlatform = `${process.platform}-${process.arch}`; - const prebuild = platformMatrix[detectedPlatform]; - if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`); - - console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`); - - const downloadDestination = `${prebuild}/nocrypto`; + const downloadDestination = `nocrypto`; const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination]; console.error(`+ tar ${unzipArgs.join(' ')}`); const unzip = child_process.spawn('tar', unzipArgs, { @@ -190,35 +149,8 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) const start = performance.now(); - let signal; - if (fastDownload) { - /** - * Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto) - * For each file it prints, we give it a deadline of 3 seconds to print the next one. - * If nothing prints after 3 seconds we exit early. - * This depends on the tar file being in order and un-tar-able in under 3sec. - */ - const controller = new AbortController(); - signal = controller.signal; - let firstMemberSeen = true; - let timeout; - unzip.stderr.on('data', chunk => { - process.stderr.write(chunk, () => { - if (firstMemberSeen) { - firstMemberSeen = false; - timeout = setTimeout(() => { - clearTimeout(timeout); - unzip.stderr.removeAllListeners('data'); - controller.abort(); - }, 3_000); - } - timeout?.refresh(); - }); - }); - } - try { - await stream.pipeline(response, unzip.stdin, { signal }); + await stream.pipeline(response, unzip.stdin); } catch { await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination)); } @@ -228,7 +160,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`); await fs.rm(nodeDepsRoot, { recursive: true, force: true }); - await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true }); + await fs.cp(resolveRoot(destination, 'nocrypto'), nodeDepsRoot, { recursive: true }); const potentialLib64Path = path.join(nodeDepsRoot, 'lib64'); try { await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib')); diff --git a/.github/scripts/utils.mjs b/.github/scripts/utils.mjs new file mode 100644 index 0000000..1a54385 --- /dev/null +++ b/.github/scripts/utils.mjs @@ -0,0 +1,89 @@ +// @ts-check + +import { execSync } from "child_process"; +import path from "path"; +import url from 'node:url'; +import { spawn } from "node:child_process"; +import { once } from "node:events"; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +/** Resolves to the root of this repository */ +export function resolveRoot(...paths) { + return path.resolve(__dirname, '..', '..', ...paths); +} + +export function getCommitFromRef(ref) { + console.error(`resolving ref: ${ref}`); + const script = resolveRoot('.github', 'scripts', 'get-commit-from-ref.sh'); + const output = execSync(`bash ${script}`, { env: { REF: ref }, encoding: 'utf-8' }) + + const regex = /COMMIT_HASH=(?[a-zA-Z0-9]+)/ + const result = regex.exec(output); + + if (!result?.groups) throw new Error('unable to parse ref.') + + const { hash } = result.groups; + + console.error(`resolved to: ${hash}`); + return hash; +} + +export function buildLibmongocryptDownloadUrl(ref, platform) { + const hash = getCommitFromRef(ref); + + // sort of a hack - if we have an official release version, it'll be in the form `major.minor`. otherwise, + // we'd expect a commit hash or `master`. + if (ref.includes('.')) { + const [major, minor, _patch] = ref.split('.'); + + // Just a note: it may appear that this logic _doesn't_ support patch releases but it actually does. + // libmongocrypt's creates release branches for minor releases in the form `r.`. + // Any patches made to this branch are committed as tags in the form ... + // So, the branch that is used for the AWS s3 upload is `r.` and the commit hash + // is the commit hash we parse from the `getCommitFromRef()` (which handles switching to git tags and + // getting the commit hash at that tag just fine). + const branch = `r${major}.${minor}` + + return `https://mciuploads.s3.amazonaws.com/libmongocrypt-release/${platform}/${branch}/${hash}/libmongocrypt.tar.gz`; + } + + // just a note here - `master` refers to the branch, the hash is the commit on that branch. + // if we ever need to download binaries from a non-master branch (or non-release branch), + // this will need to be modified somehow. + return `https://mciuploads.s3.amazonaws.com/libmongocrypt/${platform}/master/${hash}/libmongocrypt.tar.gz`; +} + +export function getLibmongocryptPrebuildName() { + const platformMatrix = { + ['darwin-arm64']: 'macos', + ['darwin-x64']: 'macos', + ['linux-ppc64']: 'rhel-71-ppc64el', + ['linux-s390x']: 'rhel72-zseries-test', + ['linux-arm64']: 'ubuntu1804-arm64', + ['linux-x64']: 'rhel-70-64-bit', + ['win32-x64']: 'windows-test' + }; + + const detectedPlatform = `${process.platform}-${process.arch}`; + const prebuild = platformMatrix[detectedPlatform]; + + if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`); + + return prebuild; +} + +/** `xtrace` style command runner, uses spawn so that stdio is inherited */ +export async function run(command, args = [], options = {}) { + const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`; + console.error(commandDetails); + const proc = spawn(command, args, { + shell: process.platform === 'win32', + stdio: 'inherit', + cwd: resolveRoot('.'), + ...options + }); + await once(proc, 'exit'); + + if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`); +} \ No newline at end of file