SERVER-130777: fix(index-builds): persist side-write multikey state for resumable index builds

jstests/noPassthrough/index_builds/resumable_index_build_multikey_from_side_writes.js

@@ -0,0 +1,116 @@
+/**
+ * Tests that a resumable index build preserves multikey information which was recorded only by the
+ * side writes interceptor (i.e. generated by writes that happened after the collection scan) when
+ * the node is cleanly shut down and the build is resumed on startup.
+ *
+ * @tags: [
+ *   # Primary-driven index builds aren't resumable.
+ *   primary_driven_index_builds_incompatible,
+ *   requires_majority_read_concern,
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
+ */
+
+import {configureFailPoint} from "jstests/libs/fail_point_util.js";
+import {funWithArgs} from "jstests/libs/parallel_shell_helpers.js";
+import {getPlanStage, getWinningPlanFromExplain} from "jstests/libs/query/analyze_plan.js";
+import {ReplSetTest} from "jstests/libs/replsettest.js";
+import {extractUUIDFromObject} from "jstests/libs/uuid_util.js";
+import {
+    IndexBuildTest,
+    ResumableIndexBuildTest,
+} from "jstests/noPassthrough/libs/index_builds/index_build.js";
+
+const dbName = "test";
+const collName = jsTestName();
+const indexName = "a_1";
+
+const rst = new ReplSetTest({nodes: 1});
+rst.startSet();
+rst.initiate();
+
+let primary = rst.getPrimary();
+let coll = primary.getDB(dbName).getCollection(collName);
+
+// Only scalar values for 'a' exist when the collection scan runs, so the bulk builder never
+// observes any multikey documents.
+assert.commandWorked(coll.insert([{a: 1}, {a: 2}]));
+
+// Hang the index build after it completes the first drain of the side writes table. At this point
+// the collection scan is finished, so any subsequent writes are captured only by the side writes
+// interceptor.
+const fp = configureFailPoint(primary, "hangAfterIndexBuildFirstDrain");
+
+const awaitCreateIndex = startParallelShell(
+    funWithArgs(
+        async function (collName, indexName) {
+            const {ResumableIndexBuildTest} = await import(
+                "jstests/noPassthrough/libs/index_builds/index_build.js"
+            );
+            ResumableIndexBuildTest.createIndexesFails(db, collName, [{a: 1}], [indexName]);
+        },
+        collName,
+        indexName,
+    ),
+    primary.port,
+);
+
+fp.wait();
+
+// This document is recorded only in the side writes table. The multikey information it generates
+// exists only in memory in the IndexBuildInterceptor and must survive the shutdown below.
+assert.commandWorked(coll.insert({a: [3, 4]}));
+
+const buildUUID = extractUUIDFromObject(
+    IndexBuildTest.assertIndexesIdHelper(coll, 1, [], [indexName], {includeBuildUUIDs: true})[
+        indexName
+    ].buildUUID,
+);
+
+clearRawMongoProgramOutput();
+
+// The failpoint is interruptible, so a clean shutdown interrupts the index build at its current
+// location and persists the resumable state.
+rst.stop(primary);
+
+// Ensure that the resumable index build state was written to disk upon clean shutdown.
+assert(RegExp("4841502.*" + buildUUID).test(rawMongoProgramOutput(".*")));
+
+rst.start(primary, {
+    noCleanData: true,
+    setParameter: {logComponentVerbosity: tojson({index: 1, storage: 1})},
+});
+primary = rst.getPrimary();
+coll = primary.getDB(dbName).getCollection(collName);
+
+awaitCreateIndex();
+
+// Ensure that the index build was resumed rather than restarted from scratch.
+checkLog.containsJson(primary, 4841700, {
+    buildUUID: function (uuid) {
+        return uuid && uuid["uuid"]["$uuid"] === buildUUID;
+    },
+});
+
+ResumableIndexBuildTest.assertCompleted(primary, coll, [buildUUID], [indexName]);
+
+// The resumed index must be marked multikey on account of the side-written array document.
+const explain = coll.find({a: 3}).hint({a: 1}).explain();
+const ixscan = getPlanStage(getWinningPlanFromExplain(explain), "IXSCAN");
+assert(ixscan.isMultiKey, "expected the resumed index to be marked multikey", {explain});
+
+// A predicate whose bounds may only be intersected on a non-multikey index must still match the
+// array document: some element is >= 4 and some element is <= 3.
+assert.eq(
+    1,
+    coll.find({a: {$gte: 4, $lte: 3}}).hint({a: 1}).itcount(),
+    "expected the array document to match on the resumed multikey index",
+);
+
+const validateRes = assert.commandWorked(coll.validate({full: true}));
+assert(validateRes.valid, "validation failed after resuming the index build", {validateRes});
+
+assert.commandWorked(coll.dropIndex(indexName));
+
+rst.stopSet();

src/mongo/db/index_builds/index_build_block.h

@@ -158,6 +158,14 @@ class IndexBuildBlock {
         }
     }
 
+    /**
+     * Returns the side writes interceptor for this index build, or null if one has not been
+     * installed (e.g. for foreground builds).
+     */
+    const IndexBuildInterceptor* getIndexBuildInterceptor() const {
+        return _indexBuildInterceptor.get();
+    }
+
 private:
     void _completeInit(OperationContext* opCtx, Collection* collection);
 

src/mongo/db/index_builds/multi_index_block.cpp

@@ -1812,10 +1812,32 @@ IndexStateInfo MultiIndexBlock::_buildIndexStateInfo(const IndexToBuild& index)
     }
 
     indexStateInfo.setSpec(index.block->getSpec());
-    indexStateInfo.setIsMultikey(index.bulk->isMultikey());
+
+    bool isMultikey = index.bulk->isMultikey();
+    MultikeyPaths multikeyPathsToPersist = index.bulk->getMultikeyPaths();
+
+    // Writes which happen after the collection scan's snapshot are recorded only by the side
+    // writes interceptor, and the multikey information they generate exists only in memory in the
+    // interceptor until commit time. The side writes table itself stores bare index keys, so
+    // draining it again after a restart cannot reconstruct this information. Fold it into the
+    // persisted multikey state; on resume it is restored into the bulk builder, which marks the
+    // index multikey at commit.
+    if (const auto* interceptor = index.block->getIndexBuildInterceptor()) {
+        if (auto sideWritesMultikeyPaths = interceptor->getMultikeyPaths()) {
+            isMultikey = true;
+            if (multikeyPathsToPersist.empty()) {
+                multikeyPathsToPersist = std::move(*sideWritesMultikeyPaths);
+            } else if (multikeyPathsToPersist.size() == sideWritesMultikeyPaths->size()) {
+                MultikeyPathTracker::mergeMultikeyPaths(&multikeyPathsToPersist,
+                                                        *sideWritesMultikeyPaths);
+            }
+        }
+    }
+
+    indexStateInfo.setIsMultikey(isMultikey);
 
     std::vector<MultikeyPath> multikeyPaths;
-    for (const auto& multikeyPath : index.bulk->getMultikeyPaths()) {
+    for (const auto& multikeyPath : multikeyPathsToPersist) {
         MultikeyPath multikeyPathObj;
         std::vector<int32_t> multikeyComponents;
         for (const auto& multikeyComponent : multikeyPath) {

src/mongo/db/index_builds/multi_index_block_test.cpp

@@ -164,7 +164,8 @@ boost::optional<std::string> findPersistedResumeState(
     OperationContext* opCtx,
     const UUID& buildUUID,
     boost::optional<IndexBuildPhaseEnum> phase = boost::none,
-    boost::optional<UUID> collectionUUID = boost::none) {
+    boost::optional<UUID> collectionUUID = boost::none,
+    ResumeIndexInfo* resumeInfoOut = nullptr) {
     boost::optional<std::string> foundResumeIdent;
     auto storageEngine = opCtx->getServiceContext()->getStorageEngine();
     const auto idents =
@@ -209,6 +210,9 @@ boost::optional<std::string> findPersistedResumeState(
         EXPECT_FALSE(cursor->next());
 
         foundResumeIdent = *it;
+        if (resumeInfoOut) {
+            *resumeInfoOut = std::move(resumeInfo);
+        }
     }
 
     return foundResumeIdent;
@@ -454,6 +458,70 @@ TEST_F(MultiIndexBlockTest, PersistResumeStateOnAbortWithoutCleanup) {
     validateTempTableIdentsOnTeardown({*indexBuildInfo1.sideWritesIdent, *indexBuildIdent});
 }
 
+// A multikey write that happens while an index build is in progress is recorded only by the side
+// writes interceptor, and the side writes table stores bare index keys. The multikey information
+// cannot be reconstructed by re-draining the table after a restart, so it must be included in the
+// persisted resume state.
+TEST_F(MultiIndexBlockTest, ResumeStateIncludesMultikeyFromSideWrites) {
+    auto indexer = getIndexer();
+    const auto buildUUID = UUID::gen();
+    indexer->setBuildUUID(buildUUID);
+
+    auto acq =
+        acquireCollection(operationContext(),
+                          CollectionAcquisitionRequest::fromOpCtx(
+                              operationContext(), getNSS(), AcquisitionPrerequisites::kWrite),
+                          MODE_X);
+    CollectionWriter coll(operationContext(), &acq);
+
+    auto storageEngine = operationContext()->getServiceContext()->getStorageEngine();
+    auto indexBuildInfo1 =
+        IndexBuildInfo(BSON("key" << BSON("a" << 1) << "name"
+                                  << "a_1"
+                                  << "v" << static_cast<int>(IndexConfig::kLatestIndexVersion)),
+                       "index-1",
+                       *storageEngine);
+
+    auto specs = unittest::assertGet(indexer->init(operationContext(),
+                                                   coll,
+                                                   {indexBuildInfo1},
+                                                   MultiIndexBlock::kNoopOnInitFn,
+                                                   MultiIndexBlock::InitMode::SteadyState,
+                                                   boost::none));
+    EXPECT_EQ(1U, specs.size());
+
+    {
+        WriteUnitOfWork wuow(operationContext());
+        ASSERT_OK(Helpers::insert(operationContext(),
+                                  acq.getCollectionPtr(),
+                                  BSON("_id" << 0 << "a" << BSON_ARRAY(1 << 2))));
+        wuow.commit();
+    }
+
+    indexer->setIsResumable(true);
+    indexer->abortWithoutCleanup(operationContext(), coll.get());
+
+    ResumeIndexInfo resumeInfo;
+    auto indexBuildIdent = findPersistedResumeState(operationContext(),
+                                                    buildUUID,
+                                                    IndexBuildPhaseEnum::kInitialized,
+                                                    coll->uuid(),
+                                                    &resumeInfo);
+    ASSERT_TRUE(indexBuildIdent);
+
+    const auto& indexes = resumeInfo.getIndexes();
+    ASSERT_EQ(1U, indexes.size());
+    ASSERT_TRUE(indexes[0].getIsMultikey());
+
+    const auto& multikeyPaths = indexes[0].getMultikeyPaths();
+    ASSERT_EQ(1U, multikeyPaths.size());
+    const auto& multikeyComponents = multikeyPaths[0].getMultikeyComponents();
+    ASSERT_EQ(1U, multikeyComponents.size());
+    ASSERT_EQ(0, multikeyComponents[0]);
+
+    validateTempTableIdentsOnTeardown({*indexBuildInfo1.sideWritesIdent, *indexBuildIdent});
+}
+
 TEST_F(MultiIndexBlockTest, PersistResumeStateOnRequestAndCommit) {
     auto indexer = getIndexer();
     const auto buildUUID = UUID::gen();