Preflight ReFS MLog recovery before applying redo

Compression.Tests/Refs/RefsMLogReplayTests.cs

@@ -0,0 +1,107 @@
+using FileSystem.Refs;
+
+namespace Compression.Tests.Refs;
+
+[TestFixture]
+public sealed class RefsMLogReplayTests {
+  [Test, Category("ErrorHandling")]
+  public void LaterUnsupportedOpcode_LeavesTargetUntouched() {
+    var target = new RecordingTarget();
+    var records = new[] {
+      LogRecord(1, RefsRedoOpcode.UpdateRow),
+      LogRecord(2, RefsRedoOpcode.ReservedUnhandled),
+    };
+
+    Assert.Throws<NotSupportedException>(() => RefsMLogRestarter.ReplaySelected(records, target));
+    Assert.That(target.Applied, Is.Empty);
+  }
+
+  [Test, Category("ErrorHandling")]
+  public void LaterUndecodedPayload_LeavesTargetUntouched() {
+    var target = new RecordingTarget { RejectLsn = 2 };
+    var records = new[] {
+      LogRecord(1, RefsRedoOpcode.UpdateRow),
+      LogRecord(2, RefsRedoOpcode.InsertRow),
+    };
+
+    Assert.Throws<NotSupportedException>(() => RefsMLogRestarter.ReplaySelected(records, target));
+    Assert.That(target.Applied, Is.Empty);
+    Assert.That(target.Preflighted, Is.EqualTo(new ulong[] { 1, 2 }));
+  }
+
+  [Test, Category("ErrorHandling")]
+  public void InvalidLastLiveChecksum_RefusesRecoveryInsteadOfReplayingPrefix() {
+    var records = new[] {
+      LogRecord(1, RefsRedoOpcode.UpdateRow),
+      LogRecord(2, RefsRedoOpcode.InsertRow),
+    };
+
+    Assert.Throws<InvalidDataException>(() => RefsMLogRecovery.SelectVerifiedForReplay(
+      records, 0, item => item.Record.Lsn != 2));
+  }
+
+  [Test, Category("HappyPath")]
+  public void InvalidStaleChecksum_DoesNotEnterCheckpointRecoveryWindow() {
+    var records = new[] {
+      LogRecord(1, RefsRedoOpcode.UpdateRow),
+      LogRecord(2, RefsRedoOpcode.InsertRow),
+    };
+    var checkedLsns = new List<ulong>();
+
+    var selected = RefsMLogRecovery.SelectVerifiedForReplay(records, 2, item => {
+      checkedLsns.Add(item.Record.Lsn);
+      return item.Record.Lsn == 2;
+    });
+
+    Assert.Multiple(() => {
+      Assert.That(checkedLsns, Is.EqualTo(new ulong[] { 2 }));
+      Assert.That(selected.Select(item => item.Record.Lsn), Is.EqualTo(new ulong[] { 2 }));
+    });
+  }
+
+  [Test, Category("HappyPath")]
+  public void ReplaySelected_AppliesOnlyRequestedRecoveryWindowInOrder() {
+    var target = new RecordingTarget();
+    var records = new[] {
+      LogRecord(0x00000001_00000001, RefsRedoOpcode.UpdateRow),
+      LogRecord(0x00000001_00000002, RefsRedoOpcode.InsertRow),
+      LogRecord(0x00000001_00000003, RefsRedoOpcode.DeleteRow),
+    };
+
+    var count = RefsMLogRestarter.ReplaySelected(records, target, 0x00000001_00000002);
+
+    Assert.Multiple(() => {
+      Assert.That(count, Is.EqualTo(2));
+      Assert.That(target.Preflighted, Is.EqualTo(new ulong[] {
+        0x00000001_00000002,
+        0x00000001_00000003,
+      }));
+      Assert.That(target.Applied, Is.EqualTo(target.Preflighted));
+    });
+  }
+
+  private static RefsMLogRecoveryRecord LogRecord(ulong lsn, RefsRedoOpcode opcode)
+    => new(0, new RefsMLogDataRecord(
+      FormatMagic: 0,
+      Lsn: lsn,
+      PreviousLsn: 0,
+      EntryChecksum: 0,
+      EntryHeaderOffset: 0,
+      PayloadOffset: 0,
+      RedoRecords: [RefsRedoRecord.Create(opcode, 0x600, [1])]),
+      []);
+
+  private sealed class RecordingTarget : IRefsRedoTarget {
+    public ulong? RejectLsn { get; init; }
+    public List<ulong> Preflighted { get; } = [];
+    public List<ulong> Applied { get; } = [];
+
+    public void Preflight(ulong lsn, RefsRedoRecord record) {
+      this.Preflighted.Add(lsn);
+      if (lsn == this.RejectLsn)
+        throw new NotSupportedException("The target does not decode this payload.");
+    }
+
+    public void Apply(ulong lsn, RefsRedoRecord record) => this.Applied.Add(lsn);
+  }
+}

Compression.Tests/Refs/RefsTests.cs

@@ -166,6 +166,27 @@ public void MLogRecovery_RejectsBrokenLiveChain() {
     Assert.Throws<InvalidDataException>(() => RefsMLogRecovery.SelectForReplay(records, 0));
   }
 
+  [Test, Category("ErrorHandling")]
+  public void MLogRecovery_RejectsMissingReferencedLiveRecord() {
+    var records = new[] {
+      RecoveryRecord(0x00000002_00000010UL, 0),
+      RecoveryRecord(0x00000002_00000012UL, 0x00000002_00000011UL),
+    };
+
+    Assert.Throws<InvalidDataException>(() =>
+      RefsMLogRecovery.SelectForReplay(records, 0x00000002_00000010UL));
+  }
+
+  [Test, Category("ErrorHandling")]
+  public void MLogRecovery_RejectsMissingFirstLivePredecessor() {
+    var records = new[] {
+      RecoveryRecord(0x00000002_00000012UL, 0x00000002_00000011UL),
+    };
+
+    Assert.Throws<InvalidDataException>(() =>
+      RefsMLogRecovery.SelectForReplay(records, 0x00000002_00000011UL));
+  }
+
   [Test, Category("HappyPath")]
   public void MLogRecovery_AllowsCircularGenerationBoundary() {
     var records = new[] {

Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Refs/RefsMLogRecovery.cs

@@ -51,13 +51,34 @@ public static IReadOnlyList<RefsMLogRecoveryRecord> Analyze(
       return [];
     }
 
-    var checksum = RefsMLogChecksum.Detect(candidates.Select(c => c.Bytes));
+    var oldest = ReadOldestRequiredLsn(image, metadata);
+    var live = candidates
+      .Where(c => oldest == 0 || CompareLsn(c.Record.Lsn, oldest) >= 0)
+      .ToArray();
+    if (live.Length == 0) {
+      checksumKind = default;
+      return [];
+    }
+
+    var checksum = RefsMLogChecksum.Detect(live.Select(c => c.Bytes));
     checksumKind = checksum.Kind;
-    var verified = candidates.Where(c => checksum.Verify(c.Bytes)).ToList();
-    if (verified.Count == 0) return [];
+    return SelectVerifiedForReplay(live, oldest, c => checksum.Verify(c.Bytes));
+  }
 
-    var oldest = ReadOldestRequiredLsn(image, metadata);
-    return SelectForReplay(verified, oldest);
+  internal static IReadOnlyList<RefsMLogRecoveryRecord> SelectVerifiedForReplay(
+      IEnumerable<RefsMLogRecoveryRecord> candidates,
+      ulong oldestRequiredLsn,
+      Func<RefsMLogRecoveryRecord, bool> verify) {
+    ArgumentNullException.ThrowIfNull(candidates);
+    ArgumentNullException.ThrowIfNull(verify);
+    var live = candidates
+      .Where(c => oldestRequiredLsn == 0 || CompareLsn(c.Record.Lsn, oldestRequiredLsn) >= 0)
+      .ToArray();
+    foreach (var candidate in live)
+      if (!verify(candidate))
+        throw new InvalidDataException(
+          $"ReFS MLog live record LSN 0x{candidate.Record.Lsn:X} failed its XOR-fold checksum.");
+    return SelectForReplay(live, oldestRequiredLsn);
   }
 
   internal static IReadOnlyList<RefsMLogRecoveryRecord> SelectForReplay(
@@ -70,14 +91,28 @@ internal static IReadOnlyList<RefsMLogRecoveryRecord> SelectForReplay(
       .ThenBy(r => LsnIndex(r.Record.Lsn))
       .ToList();
 
+    if (ordered.Count > 0) {
+      var first = ordered[0].Record;
+      if (first.PreviousLsn != 0 && CompareLsn(first.PreviousLsn, first.Lsn) >= 0)
+        throw new InvalidDataException($"ReFS MLog LSN 0x{first.Lsn:X} points to a nonpreceding LSN.");
+      if (first.PreviousLsn != 0
+          && (oldestRequiredLsn == 0 || CompareLsn(first.PreviousLsn, oldestRequiredLsn) >= 0))
+        throw new InvalidDataException(
+          $"ReFS MLog first live LSN 0x{first.Lsn:X} references missing live predecessor 0x{first.PreviousLsn:X}.");
+    }
+
     for (var i = 1; i < ordered.Count; ++i) {
       if (ordered[i - 1].Record.Lsn == ordered[i].Record.Lsn)
         throw new InvalidDataException($"ReFS MLog contains duplicate live LSN 0x{ordered[i].Record.Lsn:X}.");
 
       var current = ordered[i].Record;
       var previous = ordered[i - 1].Record;
-      if (!ShouldRequireImmediatePredecessor(current.Lsn, previous.Lsn)) continue;
-      if (current.PreviousLsn != previous.Lsn)
+      if (current.PreviousLsn != 0 && CompareLsn(current.PreviousLsn, current.Lsn) >= 0)
+        throw new InvalidDataException($"ReFS MLog LSN 0x{current.Lsn:X} points to a nonpreceding LSN.");
+      var requiresImmediate = ShouldRequireImmediatePredecessor(current.Lsn, previous.Lsn);
+      var refersToMissingLiveRecord = current.PreviousLsn != 0
+        && (oldestRequiredLsn == 0 || CompareLsn(current.PreviousLsn, oldestRequiredLsn) >= 0);
+      if (current.PreviousLsn != previous.Lsn && (requiresImmediate || refersToMissingLiveRecord))
         throw new InvalidDataException(
           $"ReFS MLog live chain is broken at LSN 0x{current.Lsn:X}: previous is 0x{current.PreviousLsn:X}, expected 0x{previous.Lsn:X}.");
     }
@@ -114,17 +149,40 @@ internal static int CompareLsn(ulong left, ulong right) {
 /// delegated to an explicit target so unknown redo grammars remain fail-closed.
 /// </summary>
 internal static class RefsMLogRestarter {
-  public static int Replay(Stream image, RefsMetadataReader metadata, IRefsRedoTarget target) {
+  public static int Replay(
+      Stream image,
+      RefsMetadataReader metadata,
+      IRefsRedoTarget target,
+      ulong minimumLsn = 0) {
     ArgumentNullException.ThrowIfNull(target);
     var recovery = RefsMLogRecovery.Analyze(image, metadata, out _);
+    return ReplaySelected(recovery, target, minimumLsn);
+  }
+
+  internal static int ReplaySelected(
+      IReadOnlyList<RefsMLogRecoveryRecord> recovery,
+      IRefsRedoTarget target,
+      ulong minimumLsn = 0) {
+    ArgumentNullException.ThrowIfNull(recovery);
+    ArgumentNullException.ThrowIfNull(target);
+    var pending = recovery
+      .Where(item => minimumLsn == 0 || RefsMLogRecovery.CompareLsn(item.Record.Lsn, minimumLsn) >= 0)
+      .SelectMany(item => item.Record.RedoRecords.Select(redo => (item.Record.Lsn, Redo: redo)))
+      .ToArray();
+
+    // An unsupported opcode or payload in a later log block must be found
+    // before a preceding block changes the target. The target validates the
+    // version-specific payload grammar; the framing layer validates opcodes.
+    foreach (var (lsn, redo) in pending) {
+      if (!Enum.IsDefined(redo.Opcode) || redo.Opcode == RefsRedoOpcode.ReservedUnhandled)
+        throw new NotSupportedException($"ReFS redo opcode 0x{(uint)redo.Opcode:X2} cannot be replayed.");
+      target.Preflight(lsn, redo);
+    }
+
     var applied = 0;
-    foreach (var item in recovery) {
-      foreach (var redo in item.Record.RedoRecords) {
-        if (redo.Opcode == RefsRedoOpcode.ReservedUnhandled)
-          throw new NotSupportedException("ReFS redo opcode 0x17 is explicitly unsupported by the native format.");
-        target.Apply(item.Record.Lsn, redo);
-        ++applied;
-      }
+    foreach (var (lsn, redo) in pending) {
+      target.Apply(lsn, redo);
+      ++applied;
     }
     return applied;
   }

Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Refs/RefsMLogWriter.cs

@@ -348,48 +348,19 @@ private static int CompareLsn(ulong left, ulong right) {
 }
 
 internal interface IRefsRedoTarget {
+  /// <summary>Checks the complete opcode payload without changing the target.</summary>
+  void Preflight(ulong lsn, RefsRedoRecord record);
   void Apply(ulong lsn, RefsRedoRecord record);
 }
 
 /// <summary>
-/// Two-pass-friendly redo enumerator. Analysis is performed by validation and
-/// LSN ordering; the target then receives redo records in durable order. Unknown
-/// opcode semantics belong to the target and must fail closed there.
+/// Compatibility entry point for the checkpoint-bounded, preflighted restarter.
 /// </summary>
 internal static class RefsMLogReplayer {
   public static int Replay(
       Stream image,
       RefsMetadataReader metadata,
       IRefsRedoTarget target,
-      ulong minimumLsn = 0) {
-    ArgumentNullException.ThrowIfNull(image);
-    ArgumentNullException.ThrowIfNull(metadata);
-    ArgumentNullException.ThrowIfNull(target);
-    if (!RefsMLogReader.TryOpen(image, metadata, out var state)) return 0;
-
-    var blocks = new List<(RefsMLogDataRecord Record, byte[] Bytes)>();
-    foreach (var offset in RefsMLogCodec.EnumerateDataBlockOffsets(state.ActiveControl, metadata.ClusterSize)) {
-      if (offset < 0 || offset > image.Length - RefsMLogCodec.LogBlockSize) continue;
-      var bytes = new byte[RefsMLogCodec.LogBlockSize];
-      image.Position = offset;
-      image.ReadExactly(bytes);
-      if (RefsMLogCodec.TryParseDataRecord(bytes, out var record)
-          && record.FormatMagic == state.ActiveControl.FormatMagic
-          && record.Lsn >= minimumLsn)
-        blocks.Add((record, bytes));
-    }
-    if (blocks.Count == 0) return 0;
-
-    var checksum = RefsMLogChecksum.Detect(blocks.Select(b => b.Bytes));
-    var applied = 0;
-    foreach (var item in blocks.OrderBy(b => b.Record.Lsn)) {
-      if (!checksum.Verify(item.Bytes))
-        throw new InvalidDataException($"ReFS MLog record LSN 0x{item.Record.Lsn:X} failed its XOR-fold checksum.");
-      foreach (var redo in item.Record.RedoRecords) {
-        target.Apply(item.Record.Lsn, redo);
-        ++applied;
-      }
-    }
-    return applied;
-  }
-}
+      ulong minimumLsn = 0)
+    => RefsMLogRestarter.Replay(image, metadata, target, minimumLsn);
+}