Skip to content

Commit

Permalink
chore(NODE-6726): make install libmongocrypt script faster (#66)
Browse files Browse the repository at this point in the history
  • Loading branch information
baileympearson authored Feb 6, 2025
1 parent 38e84d2 commit af9a3e7
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 85 deletions.
2 changes: 1 addition & 1 deletion .github/docker/Dockerfile.glibc
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ ENV PATH=$PATH:/nodejs/bin
WORKDIR /mongodb-client-encryption
COPY . .

RUN apt-get -qq update && apt-get -qq install -y python3 build-essential && ldd --version
RUN apt-get -qq update && apt-get -qq install -y python3 build-essential git && ldd --version

RUN npm run install:libmongocrypt

Expand Down
14 changes: 14 additions & 0 deletions .github/scripts/get-commit-from-ref.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#! /usr/bin/env bash
set -o errexit

git clone https://github.com/mongodb/libmongocrypt.git _libmongocrypt
cd _libmongocrypt

git checkout --detach $REF

COMMIT_HASH=$(git rev-parse HEAD)

echo "COMMIT_HASH=$COMMIT_HASH"

cd -
rm -rf _libmongocrypt
100 changes: 16 additions & 84 deletions .github/scripts/libmongocrypt.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,7 @@ import events from 'node:events';
import path from 'node:path';
import https from 'node:https';
import stream from 'node:stream/promises';
import url from 'node:url';

const __dirname = path.dirname(url.fileURLToPath(import.meta.url));

/** Resolves to the root of this repository */
function resolveRoot(...paths) {
return path.resolve(__dirname, '..', '..', ...paths);
}
import { buildLibmongocryptDownloadUrl, getLibmongocryptPrebuildName, resolveRoot, run } from './utils.mjs';

async function parseArguments() {
const pkg = JSON.parse(await fs.readFile(resolveRoot('package.json'), 'utf8'));
Expand All @@ -26,7 +19,6 @@ async function parseArguments() {
clean: { short: 'c', type: 'boolean', default: false },
build: { short: 'b', type: 'boolean', default: false },
dynamic: { type: 'boolean', default: false },
fastDownload: { type: 'boolean', default: false }, // Potentially incorrect download, only for the brave and impatient
'skip-bindings': { type: 'boolean', default: false },
help: { short: 'h', type: 'boolean', default: false }
};
Expand All @@ -46,7 +38,6 @@ async function parseArguments() {
return {
url: args.values.gitURL,
ref: args.values.libVersion,
fastDownload: args.values.fastDownload,
clean: args.values.clean,
build: args.values.build,
dynamic: args.values.dynamic,
Expand All @@ -55,26 +46,6 @@ async function parseArguments() {
};
}

/** `xtrace` style command runner, uses spawn so that stdio is inherited */
async function run(command, args = [], options = {}) {
const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
console.error(commandDetails);
const proc = child_process.spawn(command, args, {
shell: process.platform === 'win32',
stdio: 'inherit',
cwd: resolveRoot('.'),
...options
});
await events.once(proc, 'exit');

if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
}

/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
function toFlags(object) {
return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
}

export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
console.error('fetching libmongocrypt...', { url, ref });
await fs.rm(libmongocryptRoot, { recursive: true, force: true });
Expand All @@ -87,14 +58,19 @@ export async function cloneLibMongoCrypt(libmongocryptRoot, { url, ref }) {
}

export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, options) {
/** CLI flag maker: `toFlags({a: 1, b: 2})` yields `['-a=1', '-b=2']` */
function toCLIFlags(object) {
return Array.from(Object.entries(object)).map(([k, v]) => `-${k}=${v}`);
}

console.error('building libmongocrypt...');

const nodeBuildRoot = resolveRoot(nodeDepsRoot, 'tmp', 'libmongocrypt-build');

await fs.rm(nodeBuildRoot, { recursive: true, force: true });
await fs.mkdir(nodeBuildRoot, { recursive: true });

const CMAKE_FLAGS = toFlags({
const CMAKE_FLAGS = toCLIFlags({
/**
* We provide crypto hooks from Node.js binding to openssl (so disable system crypto)
* TODO: NODE-5455
Expand Down Expand Up @@ -127,12 +103,12 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option

const WINDOWS_CMAKE_FLAGS =
process.platform === 'win32' // Windows is still called "win32" when it is 64-bit
? toFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' })
? toCLIFlags({ Thost: 'x64', A: 'x64', DENABLE_WINDOWS_STATIC_RUNTIME: 'ON' })
: [];

const DARWIN_CMAKE_FLAGS =
process.platform === 'darwin' // The minimum darwin target version we want for
? toFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
? toCLIFlags({ DCMAKE_OSX_DEPLOYMENT_TARGET: '10.12' })
: [];

const cmakeProgram = process.platform === 'win32' ? 'cmake.exe' : 'cmake';
Expand All @@ -149,35 +125,18 @@ export async function buildLibMongoCrypt(libmongocryptRoot, nodeDepsRoot, option
});
}

export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload }) {
const downloadURL =
ref === 'latest'
? 'https://mciuploads.s3.amazonaws.com/libmongocrypt/all/master/latest/libmongocrypt-all.tar.gz'
: `https://mciuploads.s3.amazonaws.com/libmongocrypt/all/${ref}/libmongocrypt-all.tar.gz`;
export async function downloadLibMongoCrypt(nodeDepsRoot, { ref }) {
const prebuild = getLibmongocryptPrebuildName();

const downloadURL = buildLibmongocryptDownloadUrl(ref, prebuild);

console.error('downloading libmongocrypt...', downloadURL);
const destination = resolveRoot(`_libmongocrypt-${ref}`);

await fs.rm(destination, { recursive: true, force: true });
await fs.mkdir(destination);

const platformMatrix = {
['darwin-arm64']: 'macos',
['darwin-x64']: 'macos',
['linux-ppc64']: 'rhel-71-ppc64el',
['linux-s390x']: 'rhel72-zseries-test',
['linux-arm64']: 'ubuntu1804-arm64',
['linux-x64']: 'rhel-70-64-bit',
['win32-x64']: 'windows-test'
};

const detectedPlatform = `${process.platform}-${process.arch}`;
const prebuild = platformMatrix[detectedPlatform];
if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);

console.error(`Platform: ${detectedPlatform} Prebuild: ${prebuild}`);

const downloadDestination = `${prebuild}/nocrypto`;
const downloadDestination = `nocrypto`;
const unzipArgs = ['-xzv', '-C', `_libmongocrypt-${ref}`, downloadDestination];
console.error(`+ tar ${unzipArgs.join(' ')}`);
const unzip = child_process.spawn('tar', unzipArgs, {
Expand All @@ -190,35 +149,8 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload })

const start = performance.now();

let signal;
if (fastDownload) {
/**
* Tar will print out each file it finds inside MEMBER (ex. macos/nocrypto)
* For each file it prints, we give it a deadline of 3 seconds to print the next one.
* If nothing prints after 3 seconds we exit early.
* This depends on the tar file being in order and un-tar-able in under 3sec.
*/
const controller = new AbortController();
signal = controller.signal;
let firstMemberSeen = true;
let timeout;
unzip.stderr.on('data', chunk => {
process.stderr.write(chunk, () => {
if (firstMemberSeen) {
firstMemberSeen = false;
timeout = setTimeout(() => {
clearTimeout(timeout);
unzip.stderr.removeAllListeners('data');
controller.abort();
}, 3_000);
}
timeout?.refresh();
});
});
}

try {
await stream.pipeline(response, unzip.stdin, { signal });
await stream.pipeline(response, unzip.stdin);
} catch {
await fs.access(path.join(`_libmongocrypt-${ref}`, downloadDestination));
}
Expand All @@ -228,7 +160,7 @@ export async function downloadLibMongoCrypt(nodeDepsRoot, { ref, fastDownload })
console.error(`downloaded libmongocrypt in ${(end - start) / 1000} secs...`);

await fs.rm(nodeDepsRoot, { recursive: true, force: true });
await fs.cp(resolveRoot(destination, prebuild, 'nocrypto'), nodeDepsRoot, { recursive: true });
await fs.cp(resolveRoot(destination, 'nocrypto'), nodeDepsRoot, { recursive: true });
const potentialLib64Path = path.join(nodeDepsRoot, 'lib64');
try {
await fs.rename(potentialLib64Path, path.join(nodeDepsRoot, 'lib'));
Expand Down
89 changes: 89 additions & 0 deletions .github/scripts/utils.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// @ts-check

import { execSync } from "child_process";
import path from "path";
import url from 'node:url';
import { spawn } from "node:child_process";
import { once } from "node:events";

const __dirname = path.dirname(url.fileURLToPath(import.meta.url));

/** Resolves to the root of this repository */
export function resolveRoot(...paths) {
return path.resolve(__dirname, '..', '..', ...paths);
}

export function getCommitFromRef(ref) {
console.error(`resolving ref: ${ref}`);
const script = resolveRoot('.github', 'scripts', 'get-commit-from-ref.sh');
const output = execSync(`bash ${script}`, { env: { REF: ref }, encoding: 'utf-8' })

const regex = /COMMIT_HASH=(?<hash>[a-zA-Z0-9]+)/
const result = regex.exec(output);

if (!result?.groups) throw new Error('unable to parse ref.')

const { hash } = result.groups;

console.error(`resolved to: ${hash}`);
return hash;
}

export function buildLibmongocryptDownloadUrl(ref, platform) {
const hash = getCommitFromRef(ref);

// sort of a hack - if we have an official release version, it'll be in the form `major.minor`. otherwise,
// we'd expect a commit hash or `master`.
if (ref.includes('.')) {
const [major, minor, _patch] = ref.split('.');

// Just a note: it may appear that this logic _doesn't_ support patch releases but it actually does.
// libmongocrypt's creates release branches for minor releases in the form `r<major>.<minor>`.
// Any patches made to this branch are committed as tags in the form <major>.<minor>.<patch>.
// So, the branch that is used for the AWS s3 upload is `r<major>.<minor>` and the commit hash
// is the commit hash we parse from the `getCommitFromRef()` (which handles switching to git tags and
// getting the commit hash at that tag just fine).
const branch = `r${major}.${minor}`

return `https://mciuploads.s3.amazonaws.com/libmongocrypt-release/${platform}/${branch}/${hash}/libmongocrypt.tar.gz`;
}

// just a note here - `master` refers to the branch, the hash is the commit on that branch.
// if we ever need to download binaries from a non-master branch (or non-release branch),
// this will need to be modified somehow.
return `https://mciuploads.s3.amazonaws.com/libmongocrypt/${platform}/master/${hash}/libmongocrypt.tar.gz`;
}

export function getLibmongocryptPrebuildName() {
const platformMatrix = {
['darwin-arm64']: 'macos',
['darwin-x64']: 'macos',
['linux-ppc64']: 'rhel-71-ppc64el',
['linux-s390x']: 'rhel72-zseries-test',
['linux-arm64']: 'ubuntu1804-arm64',
['linux-x64']: 'rhel-70-64-bit',
['win32-x64']: 'windows-test'
};

const detectedPlatform = `${process.platform}-${process.arch}`;
const prebuild = platformMatrix[detectedPlatform];

if (prebuild == null) throw new Error(`Unsupported: ${detectedPlatform}`);

return prebuild;
}

/** `xtrace` style command runner, uses spawn so that stdio is inherited */
export async function run(command, args = [], options = {}) {
const commandDetails = `+ ${command} ${args.join(' ')}${options.cwd ? ` (in: ${options.cwd})` : ''}`;
console.error(commandDetails);
const proc = spawn(command, args, {
shell: process.platform === 'win32',
stdio: 'inherit',
cwd: resolveRoot('.'),
...options
});
await once(proc, 'exit');

if (proc.exitCode != 0) throw new Error(`CRASH(${proc.exitCode}): ${commandDetails}`);
}

0 comments on commit af9a3e7

Please sign in to comment.