diff --git a/lib/overlay-database-utils.js b/lib/overlay-database-utils.js index 62ca704efc..6a496333fc 100644 --- a/lib/overlay-database-utils.js +++ b/lib/overlay-database-utils.js @@ -53,6 +53,21 @@ var OverlayDatabaseMode; OverlayDatabaseMode["None"] = "none"; })(OverlayDatabaseMode || (exports.OverlayDatabaseMode = OverlayDatabaseMode = {})); exports.CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; +/** + * The maximum (uncompressed) size of the overlay base database that we will + * upload. Actions Cache has an overall capacity of 10 GB, and the Actions Cache + * client library uses zstd compression. + * + * Ideally we would apply a size limit to the compressed overlay-base database, + * but we cannot do so because compression is handled transparently by the + * Actions Cache client library. Instead we place a limit on the uncompressed + * size of the overlay-base database. + * + * Assuming 2.5:1 compression ratio, the 6 GB limit on uncompressed data would + * translate to a limit of around 2.4 GB after compression. + */ +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6000; +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000; /** * Writes a JSON file containing Git OIDs for all tracked files (represented * by path relative to the source root) under the source root. The file is @@ -192,6 +207,19 @@ async function uploadOverlayBaseDatabaseToCache(codeql, config, logger) { await codeql.databaseCleanupCluster(config, "overlay"); }); const dbLocation = config.dbLocation; + const databaseSizeBytes = await (0, util_1.tryGetFolderBytes)(dbLocation, logger); + if (databaseSizeBytes === undefined) { + logger.warning("Failed to determine database size. " + + "Skip uploading overlay-base database to cache."); + return false; + } + if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) { + const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000); + logger.warning(`Database size (${databaseSizeMB} MB) ` + + `exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` + + "Skip uploading overlay-base database to cache."); + return false; + } const codeQlVersion = (await codeql.getVersion()).version; const checkoutPath = (0, actions_util_1.getRequiredInput)("checkout_path"); const cacheKey = await generateCacheKey(config, codeQlVersion, checkoutPath); diff --git a/lib/overlay-database-utils.js.map b/lib/overlay-database-utils.js.map index 43a4b2fa89..de550c4c5e 100644 --- a/lib/overlay-database-utils.js.map +++ b/lib/overlay-database-utils.js.map @@ -1 +1 @@ -{"version":3,"file":"overlay-database-utils.js","sourceRoot":"","sources":["../src/overlay-database-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,8DAQC;AAkDD,0DAsBC;AAkCD,4DAcC;AAeD,4EAoEC;AAkBD,oFA4FC;AA9VD,uCAAyB;AACzB,2CAA6B;AAE7B,6DAA+C;AAE/C,iDAAyE;AAGzE,2CAAiE;AACjE,uCAAmD;AACnD,iCAAsE;AAEtE,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,0CAAmB,CAAA;IACnB,mDAA4B,CAAA;IAC5B,oCAAa,CAAA;AACf,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B;AAEY,QAAA,8BAA8B,GAAG,QAAQ,CAAC;AAEvD;;;;;;;;GAQG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,UAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,wBAAwB,EAAE,eAAe,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,wBAAwB,CACrC,MAAc,EACd,MAAc;IAEd,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CACzC,wBAAwB,EACxB,OAAO,CACR,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA8B,CAAC;IAC3D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,6CAA6C;YAC3C,GAAG,wBAAwB,KAAM,CAAS,CAAC,OAAO,IAAI,CAAC,EAAE,CAC5D,CAAC;QACF,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAc;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACI,KAAK,UAAU,uBAAuB,CAC3C,MAAc,EACd,UAAkB,EAClB,MAAc;IAEd,MAAM,YAAY,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,mBAAmB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACxE,MAAM,CAAC,IAAI,CACT,SAAS,YAAY,CAAC,MAAM,0BAA0B,UAAU,GAAG,CACpE,CAAC;IAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IACnE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAClC,IAAA,oCAAqB,GAAE,EACvB,sBAAsB,CACvB,CAAC;IACF,MAAM,CAAC,KAAK,CACV,oCAAoC,kBAAkB,KAAK,gBAAgB,EAAE,CAC9E,CAAC;IACF,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;IAClE,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAC1B,YAAuC,EACvC,eAA0C;IAE1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iCAAiC;AACjC,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,YAAY,GAAG,8BAA8B,CAAC;AACpD,MAAM,sBAAsB,GAAG,OAAO,CAAC,CAAC,cAAc;AAEtD;;;;;;;;GAQG;AACH,SAAgB,wBAAwB,CACtC,MAAc,EACd,MAAc,EACd,aAAqB;IAErB,uEAAuE;IACvE,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,OAAO,CACZ,GAAG,aAAa,KAAK,wBAAwB,iBAAiB,CAC/D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,MAAc,EACd,MAAc;IAEd,MAAM,mBAAmB,GAAG,MAAM,CAAC,sBAAsB,CAAC,mBAAmB,CAAC;IAC9E,IAAI,mBAAmB,KAAK,mBAAmB,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,CAAC,KAAK,CACV,4BAA4B,mBAAmB,IAAI;YACjD,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,CAAC;QAC7D,MAAM,CAAC,KAAK,CACV,wCAAwC;YACtC,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,IAAA,mBAAY,GAAE,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,CACV,8DAA8D,CAC/D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,eAAe,GAAG,wBAAwB,CAC9C,MAAM,EACN,MAAM,EACN,gDAAgD,CACjD,CAAC;IACF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yDAAyD;IACzD,MAAM,IAAA,wBAAc,EAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,MAAM,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;IAC1D,MAAM,YAAY,GAAG,IAAA,+BAAgB,EAAC,eAAe,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC7E,MAAM,CAAC,IAAI,CACT,6DAA6D,QAAQ,EAAE,CACxE,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAW,EAC/B,sBAAsB,EACtB,YAAY,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,EAC9C,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;QACF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;YAClE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,CACZ,mDAAmD;YACjD,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,oDAAoD,UAAU,EAAE,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAOD;;;;;;;;;;GAUG;AACI,KAAK,UAAU,oCAAoC,CACxD,MAAc,EACd,MAAc,EACd,MAAc;IAEd,MAAM,mBAAmB,GAAG,MAAM,CAAC,sBAAsB,CAAC,mBAAmB,CAAC;IAC9E,IAAI,mBAAmB,KAAK,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CACV,4BAA4B,mBAAmB,IAAI;YACjD,oDAAoD,CACvD,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,CAAC;QAC7D,MAAM,CAAC,KAAK,CACV,wCAAwC;YACtC,oDAAoD,CACvD,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,IAAA,mBAAY,GAAE,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,CACV,kEAAkE,CACnE,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,MAAM,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;IAC1D,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAE7D,MAAM,CAAC,IAAI,CACT,uEAAuE,UAAU,EAAE,CACpF,CAAC;IAEF,IAAI,0BAA0B,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,IAAA,kBAAW,EAChC,sBAAsB,EACtB,YAAY,CAAC,YAAY,CAAC,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,EACnD,GAAG,EAAE;YACH,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACxE,CAAC,CACF,CAAC;QACF,0BAA0B,GAAG,IAAI,CAAC,KAAK,CACrC,WAAW,CAAC,GAAG,EAAE,GAAG,qBAAqB,CAC1C,CAAC;QAEF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,CAAC,IAAI,CACT,sDAAsD,QAAQ,EAAE,CACjE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,CACZ,uDAAuD;YACrD,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9D,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,wBAAwB,CAC9C,MAAM,EACN,MAAM,EACN,6CAA6C,CAC9C,CAAC;IACF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAiB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CACT,mEAAmE,CACpE,CAAC;QACF,0EAA0E;QAC1E,sEAAsE;QACtE,qEAAqE;QACrE,uDAAuD;QACvD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,oDAAoD,UAAU,EAAE,CAAC,CAAC;IAC9E,OAAO;QACL,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAChD,0BAA0B;KAC3B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,MAAc,EACd,aAAqB,EACrB,YAAoB;IAEpB,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAY,EAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,GAAG,kBAAkB,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,GAAG,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc,EAAE,aAAqB;IAC/D,6EAA6E;IAC7E,0EAA0E;IAC1E,yDAAyD;IACzD,EAAE;IACF,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,0EAA0E;IAC1E,qEAAqE;IACrE,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,OAAO,GAAG,YAAY,IAAI,aAAa,IAAI,SAAS,IAAI,aAAa,GAAG,CAAC;AAC3E,CAAC"} \ No newline at end of file +{"version":3,"file":"overlay-database-utils.js","sourceRoot":"","sources":["../src/overlay-database-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,8DAQC;AAkDD,0DAsBC;AAkCD,4DAcC;AAeD,4EAwFC;AAkBD,oFA4FC;AAnYD,uCAAyB;AACzB,2CAA6B;AAE7B,6DAA+C;AAE/C,iDAAyE;AAGzE,2CAAiE;AACjE,uCAAmD;AACnD,iCAAsE;AAEtE,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,0CAAmB,CAAA;IACnB,mDAA4B,CAAA;IAC5B,oCAAa,CAAA;AACf,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B;AAEY,QAAA,8BAA8B,GAAG,QAAQ,CAAC;AAEvD;;;;;;;;;;;;GAYG;AACH,MAAM,wCAAwC,GAAG,IAAI,CAAC;AACtD,MAAM,2CAA2C,GAC/C,wCAAwC,GAAG,SAAS,CAAC;AAEvD;;;;;;;;GAQG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,UAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,wBAAwB,EAAE,eAAe,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,wBAAwB,CACrC,MAAc,EACd,MAAc;IAEd,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CACzC,wBAAwB,EACxB,OAAO,CACR,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA8B,CAAC;IAC3D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,6CAA6C;YAC3C,GAAG,wBAAwB,KAAM,CAAS,CAAC,OAAO,IAAI,CAAC,EAAE,CAC5D,CAAC;QACF,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAc;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACI,KAAK,UAAU,uBAAuB,CAC3C,MAAc,EACd,UAAkB,EAClB,MAAc;IAEd,MAAM,YAAY,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,mBAAmB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACxE,MAAM,CAAC,IAAI,CACT,SAAS,YAAY,CAAC,MAAM,0BAA0B,UAAU,GAAG,CACpE,CAAC;IAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IACnE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAClC,IAAA,oCAAqB,GAAE,EACvB,sBAAsB,CACvB,CAAC;IACF,MAAM,CAAC,KAAK,CACV,oCAAoC,kBAAkB,KAAK,gBAAgB,EAAE,CAC9E,CAAC;IACF,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;IAClE,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAC1B,YAAuC,EACvC,eAA0C;IAE1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iCAAiC;AACjC,MAAM,aAAa,GAAG,CAAC,CAAC;AACxB,MAAM,YAAY,GAAG,8BAA8B,CAAC;AACpD,MAAM,sBAAsB,GAAG,OAAO,CAAC,CAAC,cAAc;AAEtD;;;;;;;;GAQG;AACH,SAAgB,wBAAwB,CACtC,MAAc,EACd,MAAc,EACd,aAAqB;IAErB,uEAAuE;IACvE,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,wBAAwB,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,OAAO,CACZ,GAAG,aAAa,KAAK,wBAAwB,iBAAiB,CAC/D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,gCAAgC,CACpD,MAAc,EACd,MAAc,EACd,MAAc;IAEd,MAAM,mBAAmB,GAAG,MAAM,CAAC,sBAAsB,CAAC,mBAAmB,CAAC;IAC9E,IAAI,mBAAmB,KAAK,mBAAmB,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,CAAC,KAAK,CACV,4BAA4B,mBAAmB,IAAI;YACjD,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,CAAC;QAC7D,MAAM,CAAC,KAAK,CACV,wCAAwC;YACtC,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,IAAA,mBAAY,GAAE,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,CACV,8DAA8D,CAC/D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,eAAe,GAAG,wBAAwB,CAC9C,MAAM,EACN,MAAM,EACN,gDAAgD,CACjD,CAAC;IACF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,yDAAyD;IACzD,MAAM,IAAA,wBAAc,EAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IAErC,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAiB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,OAAO,CACZ,qCAAqC;YACnC,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,iBAAiB,GAAG,2CAA2C,EAAE,CAAC;QACpE,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC;QACjE,MAAM,CAAC,OAAO,CACZ,kBAAkB,cAAc,OAAO;YACrC,gCAAgC,wCAAwC,QAAQ;YAChF,gDAAgD,CACnD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;IAC1D,MAAM,YAAY,GAAG,IAAA,+BAAgB,EAAC,eAAe,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC7E,MAAM,CAAC,IAAI,CACT,6DAA6D,QAAQ,EAAE,CACxE,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAW,EAC/B,sBAAsB,EACtB,YAAY,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,EAC9C,GAAG,EAAE,GAAE,CAAC,CACT,CAAC;QACF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;YAClE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,CACZ,mDAAmD;YACjD,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9D,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,oDAAoD,UAAU,EAAE,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAOD;;;;;;;;;;GAUG;AACI,KAAK,UAAU,oCAAoC,CACxD,MAAc,EACd,MAAc,EACd,MAAc;IAEd,MAAM,mBAAmB,GAAG,MAAM,CAAC,sBAAsB,CAAC,mBAAmB,CAAC;IAC9E,IAAI,mBAAmB,KAAK,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CACV,4BAA4B,mBAAmB,IAAI;YACjD,oDAAoD,CACvD,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,CAAC;QAC7D,MAAM,CAAC,KAAK,CACV,wCAAwC;YACtC,oDAAoD,CACvD,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,IAAA,mBAAY,GAAE,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,CACV,kEAAkE,CACnE,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACrC,MAAM,aAAa,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC;IAC1D,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAE7D,MAAM,CAAC,IAAI,CACT,uEAAuE,UAAU,EAAE,CACpF,CAAC;IAEF,IAAI,0BAA0B,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,qBAAqB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,IAAA,kBAAW,EAChC,sBAAsB,EACtB,YAAY,CAAC,YAAY,CAAC,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,EACnD,GAAG,EAAE;YACH,MAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;QACxE,CAAC,CACF,CAAC;QACF,0BAA0B,GAAG,IAAI,CAAC,KAAK,CACrC,WAAW,CAAC,GAAG,EAAE,GAAG,qBAAqB,CAC1C,CAAC;QAEF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,CAAC,IAAI,CACT,sDAAsD,QAAQ,EAAE,CACjE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,CACZ,uDAAuD;YACrD,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAC9D,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,wBAAwB,CAC9C,MAAM,EACN,MAAM,EACN,6CAA6C,CAC9C,CAAC;IACF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC;QACrE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,IAAA,wBAAiB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACtE,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CACT,mEAAmE,CACpE,CAAC;QACF,0EAA0E;QAC1E,sEAAsE;QACtE,qEAAqE;QACrE,uDAAuD;QACvD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,oDAAoD,UAAU,EAAE,CAAC,CAAC;IAC9E,OAAO;QACL,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAChD,0BAA0B;KAC3B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,MAAc,EACd,aAAqB,EACrB,YAAoB;IAEpB,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAY,EAAC,YAAY,CAAC,CAAC;IAC7C,OAAO,GAAG,kBAAkB,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,GAAG,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc,EAAE,aAAqB;IAC/D,6EAA6E;IAC7E,0EAA0E;IAC1E,yDAAyD;IACzD,EAAE;IACF,6EAA6E;IAC7E,4EAA4E;IAC5E,6EAA6E;IAC7E,0EAA0E;IAC1E,qEAAqE;IACrE,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,OAAO,GAAG,YAAY,IAAI,aAAa,IAAI,SAAS,IAAI,aAAa,GAAG,CAAC;AAC3E,CAAC"} \ No newline at end of file diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index c73fe3f581..a01546f067 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -18,6 +18,23 @@ export enum OverlayDatabaseMode { export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; +/** + * The maximum (uncompressed) size of the overlay base database that we will + * upload. Actions Cache has an overall capacity of 10 GB, and the Actions Cache + * client library uses zstd compression. + * + * Ideally we would apply a size limit to the compressed overlay-base database, + * but we cannot do so because compression is handled transparently by the + * Actions Cache client library. Instead we place a limit on the uncompressed + * size of the overlay-base database. + * + * Assuming 2.5:1 compression ratio, the 6 GB limit on uncompressed data would + * translate to a limit of around 2.4 GB after compression. + */ +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6000; +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = + OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000; + /** * Writes a JSON file containing Git OIDs for all tracked files (represented * by path relative to the source root) under the source root. The file is @@ -212,6 +229,26 @@ export async function uploadOverlayBaseDatabaseToCache( }); const dbLocation = config.dbLocation; + + const databaseSizeBytes = await tryGetFolderBytes(dbLocation, logger); + if (databaseSizeBytes === undefined) { + logger.warning( + "Failed to determine database size. " + + "Skip uploading overlay-base database to cache.", + ); + return false; + } + + if (databaseSizeBytes > OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES) { + const databaseSizeMB = Math.round(databaseSizeBytes / 1_000_000); + logger.warning( + `Database size (${databaseSizeMB} MB) ` + + `exceeds maximum upload size (${OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB} MB). ` + + "Skip uploading overlay-base database to cache.", + ); + return false; + } + const codeQlVersion = (await codeql.getVersion()).version; const checkoutPath = getRequiredInput("checkout_path"); const cacheKey = await generateCacheKey(config, codeQlVersion, checkoutPath);