[ntuple] automatic evolution from T --> unique_ptr/optional<T>

Author: jblomer

tree/ntuple/inc/ROOT/RField/RFieldSTLMisc.hxx

@@ -195,6 +195,9 @@ class RNullableField : public RFieldBase {
    ROOT::Internal::RColumnIndex fNWritten{0};
 
 protected:
+   // For reading, indicates that we read type T as a nullable field of type T, i.e. the value is always present
+   bool fIsEvolvedFromInnerType = false;
+
    const RFieldBase::RColumnRepresentations &GetColumnRepresentations() const final;
    void GenerateColumns() final;
    void GenerateColumns(const ROOT::RNTupleDescriptor &) final;

tree/ntuple/src/RField.cxx

@@ -797,7 +797,7 @@ const ROOT::RFieldBase::RColumnRepresentations &ROOT::RNullableField::GetColumnR
                                                   {ENTupleColumnType::kIndex64},
                                                   {ENTupleColumnType::kSplitIndex32},
                                                   {ENTupleColumnType::kIndex32}},
-                                                 {});
+                                                 {{}});
    return representations;
 }
 
@@ -808,7 +808,8 @@ void ROOT::RNullableField::GenerateColumns()
 
 void ROOT::RNullableField::GenerateColumns(const ROOT::RNTupleDescriptor &desc)
 {
-   GenerateColumnsImpl<ROOT::Internal::RColumnIndex>(desc);
+   if (!fIsEvolvedFromInnerType)
+      GenerateColumnsImpl<ROOT::Internal::RColumnIndex>(desc);
 }
 
 std::size_t ROOT::RNullableField::AppendNull()
@@ -830,8 +831,13 @@ void ROOT::RNullableField::ReconcileOnDiskField(const RNTupleDescriptor &desc)
    static const std::vector<std::string> prefixes = {"std::optional<", "std::unique_ptr<"};
 
    const auto &fieldDesc = desc.GetFieldDescriptor(GetOnDiskId());
-   EnsureMatchingOnDiskField(fieldDesc, kDiffTypeName);
-   EnsureMatchingTypePrefix(fieldDesc, prefixes);
+   try {
+      EnsureMatchingOnDiskField(fieldDesc, kDiffTypeName);
+      EnsureMatchingTypePrefix(fieldDesc, prefixes);
+   } catch (const RException &) {
+      fSubfields[0]->SetOnDiskId(GetOnDiskId());
+      fIsEvolvedFromInnerType = true;
+   }
 }
 
 ROOT::RNTupleLocalIndex ROOT::RNullableField::GetItemIndex(ROOT::NTupleSize_t globalIndex)
@@ -900,16 +906,28 @@ void *ROOT::RUniquePtrField::PrepareRead(void *to, bool hasOnDiskValue)
 
 void ROOT::RUniquePtrField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to)
 {
-   auto itemIndex = GetItemIndex(globalIndex);
-   const bool hasOnDiskValue = itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
+   RNTupleLocalIndex itemIndex;
+   if (!fIsEvolvedFromInnerType)
+      itemIndex = GetItemIndex(globalIndex);
+   const bool hasOnDiskValue = fIsEvolvedFromInnerType || itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
    auto valuePtr = PrepareRead(to, hasOnDiskValue);
-   if (hasOnDiskValue)
-      CallReadOn(*fSubfields[0], itemIndex, valuePtr);
+   if (hasOnDiskValue) {
+      if (fIsEvolvedFromInnerType) {
+         CallReadOn(*fSubfields[0], globalIndex, valuePtr);
+      } else {
+         CallReadOn(*fSubfields[0], itemIndex, valuePtr);
+      }
+   }
 }
 
 void ROOT::RUniquePtrField::ReadInClusterImpl(ROOT::RNTupleLocalIndex localIndex, void *to)
 {
-   auto itemIndex = GetItemIndex(localIndex);
+   RNTupleLocalIndex itemIndex;
+   if (!fIsEvolvedFromInnerType) {
+      itemIndex = GetItemIndex(localIndex);
+   } else {
+      itemIndex = localIndex;
+   }
    const bool hasOnDiskValue = itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
    auto valuePtr = PrepareRead(to, hasOnDiskValue);
    if (hasOnDiskValue)
@@ -992,16 +1010,28 @@ void ROOT::ROptionalField::PrepareRead(void *to, bool hasOnDiskValue)
 
 void ROOT::ROptionalField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to)
 {
-   auto itemIndex = GetItemIndex(globalIndex);
-   const bool hasOnDiskValue = itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
+   RNTupleLocalIndex itemIndex;
+   if (!fIsEvolvedFromInnerType)
+      itemIndex = GetItemIndex(globalIndex);
+   const bool hasOnDiskValue = fIsEvolvedFromInnerType || itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
    PrepareRead(to, hasOnDiskValue);
-   if (hasOnDiskValue)
-      CallReadOn(*fSubfields[0], itemIndex, to);
+   if (hasOnDiskValue) {
+      if (fIsEvolvedFromInnerType) {
+         CallReadOn(*fSubfields[0], globalIndex, to);
+      } else {
+         CallReadOn(*fSubfields[0], itemIndex, to);
+      }
+   }
 }
 
 void ROOT::ROptionalField::ReadInClusterImpl(ROOT::RNTupleLocalIndex localIndex, void *to)
 {
-   auto itemIndex = GetItemIndex(localIndex);
+   RNTupleLocalIndex itemIndex;
+   if (!fIsEvolvedFromInnerType) {
+      itemIndex = GetItemIndex(localIndex);
+   } else {
+      itemIndex = localIndex;
+   }
    const bool hasOnDiskValue = itemIndex.GetIndexInCluster() != ROOT::kInvalidNTupleIndex;
    PrepareRead(to, hasOnDiskValue);
    if (hasOnDiskValue)

tree/ntuple/test/ntuple_evolution_type.cxx

@@ -207,3 +207,32 @@ TEST(RNTupleEvolution, CheckPairTuple)
    EXPECT_EQ(1, std::get<0>(p(0)));
    EXPECT_DOUBLE_EQ(2.0, std::get<1>(p(0)));
 }
+
+TEST(RNTupleEvolution, CheckNullable)
+{
+   FileRaii fileGuard("test_ntuple_evolution_check_nullable.root");
+   {
+      auto model = ROOT::RNTupleModel::Create();
+      auto o = model->MakeField<std::optional<std::int32_t>>("o");
+      auto u = model->MakeField<std::unique_ptr<std::int32_t>>("u");
+      auto i = model->MakeField<std::int32_t>("i");
+      auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
+
+      *o = 7;
+      *u = std::make_unique<std::int32_t>(11);
+      *i = 13;
+      writer->Fill();
+   }
+
+   auto reader = RNTupleReader::Open("ntpl", fileGuard.GetPath());
+
+   auto v1 = reader->GetView<std::unique_ptr<std::int64_t>>("o");
+   auto v2 = reader->GetView<std::optional<std::int64_t>>("u");
+   auto v3 = reader->GetView<std::unique_ptr<std::int64_t>>("i");
+   auto v4 = reader->GetView<std::optional<std::int64_t>>("i");
+
+   EXPECT_EQ(7, *v1(0));
+   EXPECT_EQ(11, *v2(0));
+   EXPECT_EQ(13, *v3(0));
+   EXPECT_EQ(13, *v4(0));
+}