Parse native TUX3 allocation trees and journal metadata

Compression.Tests/Tux3/Tux3Tests.cs

@@ -7,6 +7,8 @@ namespace Compression.Tests.Tux3;
 
 [TestFixture]
 public class Tux3Tests {
+  private const int NativeBlockSize = 4096;
+
   private static byte[] BuildNativeImage(
     bool legacy2012 = false,
     ulong volBlocks = 0x1234,
@@ -32,6 +34,117 @@ private static byte[] BuildNativeImage(
     return image;
   }
 
+  /// <summary>
+  /// Builds a small but structurally native metadata graph:
+  /// superblock -> two-level inode btree -> bitmap inode -> one-level data dleaf -> bitmap data.
+  /// Optional log payloads are written oldest-to-newest to consecutive physical blocks starting at 6.
+  /// </summary>
+  private static byte[] BuildAllocationMappedImage(
+    IReadOnlyList<byte[]>? logPayloads = null,
+    params int[] additionallyAllocated) {
+    const int volumeBlocks = 16;
+    logPayloads ??= [];
+    var image = BuildNativeImage(volBlocks: volumeBlocks, imageLength: volumeBlocks * NativeBlockSize);
+    var super = image.AsSpan(Tux3Reader.SuperblockOffset, Tux3Reader.DiskSuperSize);
+
+    // inode-tree root: depth 2, root bnode at physical block 2.
+    BinaryPrimitives.WriteUInt64BigEndian(super.Slice(0x28, 8), PackRoot(direct: false, countOrDepth: 2, block: 2));
+
+    // Root bnode with one leftmost entry leading to the inode leaf.
+    var bnode = GetBlock(image, 2);
+    BinaryPrimitives.WriteUInt16BigEndian(bnode.Slice(0, 2), 0xB4DE);
+    BinaryPrimitives.WriteUInt32BigEndian(bnode.Slice(4, 4), 1);
+    BinaryPrimitives.WriteUInt64BigEndian(bnode.Slice(8, 8), 0);
+    BinaryPrimitives.WriteUInt64BigEndian(bnode.Slice(16, 8), 3);
+
+    // ileaf covers inode 0 and bitmap inode 1. Inode 0 is empty; inode 1 has only DATA_BTREE.
+    var ileaf = GetBlock(image, 3);
+    BinaryPrimitives.WriteUInt16BigEndian(ileaf.Slice(0, 2), 0x90DE);
+    BinaryPrimitives.WriteUInt16BigEndian(ileaf.Slice(2, 2), 2);
+    BinaryPrimitives.WriteUInt64BigEndian(ileaf.Slice(8, 8), 0);
+    BinaryPrimitives.WriteUInt16BigEndian(ileaf.Slice(16, 2), 0x2000); // DATA_BTREE, version 0
+    BinaryPrimitives.WriteUInt64BigEndian(ileaf.Slice(18, 8), PackRoot(direct: false, countOrDepth: 1, block: 4));
+    BinaryPrimitives.WriteUInt16BigEndian(ileaf.Slice(NativeBlockSize - 2, 2), 0);
+    BinaryPrimitives.WriteUInt16BigEndian(ileaf.Slice(NativeBlockSize - 4, 2), 10);
+
+    // Bitmap data tree leaf: logical bitmap block 0 is physical block 5, followed by a hole sentinel.
+    var dleaf = GetBlock(image, 4);
+    BinaryPrimitives.WriteUInt16BigEndian(dleaf.Slice(0, 2), 0xBEAF);
+    BinaryPrimitives.WriteUInt16BigEndian(dleaf.Slice(2, 2), 2);
+    BinaryPrimitives.WriteUInt64BigEndian(dleaf.Slice(8, 8), 0);
+    BinaryPrimitives.WriteUInt64BigEndian(dleaf.Slice(16, 8), 5);
+    BinaryPrimitives.WriteUInt64BigEndian(dleaf.Slice(24, 8), 1);
+    BinaryPrimitives.WriteUInt64BigEndian(dleaf.Slice(32, 8), 0);
+
+    var allocated = new HashSet<int> { 0, 1, 2, 3, 4, 5 };
+    foreach (var block in additionallyAllocated)
+      allocated.Add(block);
+
+    for (var i = 0; i < logPayloads.Count; ++i) {
+      var physical = 6 + i;
+      allocated.Add(physical);
+      var previous = i == 0 ? 0UL : (ulong)(physical - 1);
+      WriteLogBlock(GetBlock(image, physical), previous, logPayloads[i]);
+    }
+
+    BinaryPrimitives.WriteUInt64BigEndian(
+      super.Slice(0x58, 8),
+      logPayloads.Count == 0 ? 0UL : (ulong)(5 + logPayloads.Count));
+    BinaryPrimitives.WriteUInt32BigEndian(super.Slice(0x60, 4), (uint)logPayloads.Count);
+
+    var bitmap = GetBlock(image, 5);
+    foreach (var block in allocated)
+      bitmap[block >> 3] |= (byte)(1 << (block & 7));
+
+    return image;
+  }
+
+  private static Span<byte> GetBlock(byte[] image, int block)
+    => image.AsSpan(block * NativeBlockSize, NativeBlockSize);
+
+  private static ulong PackRoot(bool direct, ushort countOrDepth, ulong block)
+    => (direct ? 1UL << 63 : 0) | ((ulong)countOrDepth << 48) | block;
+
+  private static void WriteLogBlock(Span<byte> block, ulong previous, ReadOnlySpan<byte> payload) {
+    BinaryPrimitives.WriteUInt16BigEndian(block.Slice(0, 2), 0x10AD);
+    BinaryPrimitives.WriteUInt16BigEndian(block.Slice(2, 2), checked((ushort)payload.Length));
+    BinaryPrimitives.WriteUInt64BigEndian(block.Slice(8, 8), previous);
+    payload.CopyTo(block[16..]);
+  }
+
+  private static byte[] AllocationLog(Tux3JournalRecordType type, uint count, ulong block) {
+    var result = new byte[11];
+    result[0] = (byte)type;
+    BinaryPrimitives.WriteUInt32BigEndian(result.AsSpan(1, 4), count);
+    WriteUInt48BigEndian(result.AsSpan(5, 6), block);
+    return result;
+  }
+
+  private static byte[] BNodeAddLog(ulong parent, ulong child, ulong key) {
+    var result = new byte[19];
+    result[0] = (byte)Tux3JournalRecordType.BNodeAdd;
+    WriteUInt48BigEndian(result.AsSpan(1, 6), parent);
+    WriteUInt48BigEndian(result.AsSpan(7, 6), child);
+    WriteUInt48BigEndian(result.AsSpan(13, 6), key);
+    return result;
+  }
+
+  private static byte[] Concat(params byte[][] parts) {
+    var result = new byte[parts.Sum(static part => part.Length)];
+    var offset = 0;
+    foreach (var part in parts) {
+      part.CopyTo(result, offset);
+      offset += part.Length;
+    }
+    return result;
+  }
+
+  private static void WriteUInt48BigEndian(Span<byte> destination, ulong value) {
+    Assert.That(value, Is.LessThan(1UL << 48));
+    BinaryPrimitives.WriteUInt16BigEndian(destination[..2], (ushort)(value >> 32));
+    BinaryPrimitives.WriteUInt32BigEndian(destination.Slice(2, 4), (uint)value);
+  }
+
   [Test, Category("Spec")]
   public void Reader_ParsesCanonicalPackedBigEndianDiskSuper() {
     var image = BuildNativeImage();
@@ -54,6 +167,7 @@ public void Reader_ParsesCanonicalPackedBigEndianDiskSuper() {
       Assert.That(reader.AtomGeneration, Is.EqualTo(0x5060_7080U));
       Assert.That(reader.LogChain, Is.EqualTo(0x2222UL));
       Assert.That(reader.LogCount, Is.EqualTo(3U));
+      Assert.That(reader.AllocationMapValid, Is.False);
     });
 
     Assert.That(reader.Entries.Select(entry => entry.Name),
@@ -123,6 +237,142 @@ public void Descriptor_ExtractsNativeMetadataWithoutInventedFileTable() {
     }
   }
 
+  [Test, Category("Spec")]
+  public void AllocationTree_ParsesBNodeILeafDLeafAndBitmapRuns() {
+    var image = BuildAllocationMappedImage();
+    using var stream = new MemoryStream(image, writable: false);
+    using var reader = new Tux3Reader(stream);
+
+    Assert.Multiple(() => {
+      Assert.That(reader.JournalValid, Is.True);
+      Assert.That(reader.JournalRecords, Is.Empty);
+      Assert.That(reader.AllocationMapValid, Is.True);
+      Assert.That(reader.NativeMetadataStatus, Is.EqualTo("allocation-map+journal"));
+      Assert.That(reader.AllocationRuns, Is.EqualTo(new[] {
+        new Tux3BlockRun(0, 6, true),
+        new Tux3BlockRun(6, 10, false),
+      }));
+    });
+
+    using var layoutStream = new MemoryStream(image, writable: false);
+    var extents = new Tux3FormatDescriptor().EnumerateExtents(layoutStream).ToArray();
+    Assert.That(extents, Has.Length.EqualTo(2));
+    Assert.Multiple(() => {
+      Assert.That(extents[0].Offset, Is.Zero);
+      Assert.That(extents[0].Length, Is.EqualTo(6L * NativeBlockSize));
+      Assert.That(extents[0].Kind, Is.EqualTo(DefragBlockKind.MetadataReserved));
+      Assert.That(extents[1].Offset, Is.EqualTo(6L * NativeBlockSize));
+      Assert.That(extents[1].Length, Is.EqualTo(10L * NativeBlockSize));
+      Assert.That(extents[1].Kind, Is.EqualTo(DefragBlockKind.Free));
+    });
+  }
+
+  [Test, Category("Spec")]
+  public void AllocationJournal_ReplaysAllocationOnlyRecordsForEffectiveBitmap() {
+    var payload = Concat(
+      AllocationLog(Tux3JournalRecordType.BlockAllocate, 1, 7),
+      AllocationLog(Tux3JournalRecordType.BlockFree, 1, 8));
+    var image = BuildAllocationMappedImage([payload], 8);
+    using var stream = new MemoryStream(image, writable: false);
+    using var reader = new Tux3Reader(stream);
+
+    Assert.Multiple(() => {
+      Assert.That(reader.JournalValid, Is.True);
+      Assert.That(reader.JournalRecords.Select(static record => record.Type), Is.EqualTo(new[] {
+        Tux3JournalRecordType.BlockAllocate,
+        Tux3JournalRecordType.BlockFree,
+      }));
+      Assert.That(reader.JournalRecords[0].Block, Is.EqualTo(7));
+      Assert.That(reader.JournalRecords[0].Count, Is.EqualTo(1));
+      Assert.That(reader.AllocationMapValid, Is.True);
+      Assert.That(reader.AllocationRuns, Is.EqualTo(new[] {
+        new Tux3BlockRun(0, 8, true),
+        new Tux3BlockRun(8, 8, false),
+      }));
+    });
+  }
+
+  [Test, Category("HappyPath")]
+  public void WipeUnusedSpace_UsesEffectiveNativeBitmapWithoutTouchingAllocatedBlocks() {
+    var payload = Concat(
+      AllocationLog(Tux3JournalRecordType.BlockAllocate, 1, 7),
+      AllocationLog(Tux3JournalRecordType.BlockFree, 1, 8));
+    var image = BuildAllocationMappedImage([payload], 8);
+    image[7 * NativeBlockSize] = 0x5A;
+    Array.Fill(image, (byte)0xA5, 8 * NativeBlockSize, 8 * NativeBlockSize);
+    using var stream = new MemoryStream(image, writable: true);
+
+    var wiped = ((IWipeEmpty)new Tux3FormatDescriptor()).WipeUnusedSpace(
+      stream,
+      wipeClusterTips: false,
+      wipeDeletedEntries: false);
+
+    Assert.Multiple(() => {
+      Assert.That(wiped, Is.EqualTo(8L * NativeBlockSize));
+      Assert.That(image[7 * NativeBlockSize], Is.EqualTo(0x5A));
+      Assert.That(image.AsSpan(8 * NativeBlockSize, 8 * NativeBlockSize).ToArray(), Is.All.Zero);
+    });
+  }
+
+  [Test, Category("Regression")]
+  public void ActiveStructuralJournalRecord_DisablesAllocationMapAndWipe() {
+    var image = BuildAllocationMappedImage([BNodeAddLog(2, 3, 0)]);
+    var original = image.ToArray();
+    using var readerStream = new MemoryStream(image, writable: false);
+    using var reader = new Tux3Reader(readerStream);
+
+    Assert.Multiple(() => {
+      Assert.That(reader.JournalValid, Is.True);
+      Assert.That(reader.JournalRecords.Single().Type, Is.EqualTo(Tux3JournalRecordType.BNodeAdd));
+      Assert.That(reader.AllocationMapValid, Is.False);
+      Assert.That(reader.NativeMetadataStatus, Is.EqualTo("journal-needs-structural-replay"));
+    });
+
+    using var wipeStream = new MemoryStream(image, writable: true);
+    var wiped = ((IWipeEmpty)new Tux3FormatDescriptor()).WipeUnusedSpace(
+      wipeStream,
+      wipeClusterTips: false,
+      wipeDeletedEntries: false);
+    Assert.Multiple(() => {
+      Assert.That(wiped, Is.Zero);
+      Assert.That(image, Is.EqualTo(original));
+    });
+  }
+
+  [Test, Category("Regression")]
+  public void LatestUnify_MakesOlderStructuralLogRecordsInactive() {
+    var newer = Concat(
+      [(byte)Tux3JournalRecordType.Unify],
+      AllocationLog(Tux3JournalRecordType.BlockAllocate, 1, 9));
+    var image = BuildAllocationMappedImage([BNodeAddLog(2, 3, 0), newer]);
+    using var stream = new MemoryStream(image, writable: false);
+    using var reader = new Tux3Reader(stream);
+
+    Assert.Multiple(() => {
+      Assert.That(reader.JournalValid, Is.True);
+      Assert.That(reader.JournalRecords.Select(static record => record.Type), Is.EqualTo(new[] {
+        Tux3JournalRecordType.BNodeAdd,
+        Tux3JournalRecordType.Unify,
+        Tux3JournalRecordType.BlockAllocate,
+      }));
+      Assert.That(reader.AllocationMapValid, Is.True);
+      Assert.That(reader.AllocationRuns.Any(run => run.IsAllocated && run.StartBlock <= 9 && 9 < run.StartBlock + run.BlockCount), Is.True);
+    });
+  }
+
+  [Test, Category("Regression")]
+  public void MalformedJournal_DisablesAllocationMap() {
+    var image = BuildAllocationMappedImage([[0x99]]);
+    using var stream = new MemoryStream(image, writable: false);
+    using var reader = new Tux3Reader(stream);
+
+    Assert.Multiple(() => {
+      Assert.That(reader.JournalValid, Is.False);
+      Assert.That(reader.AllocationMapValid, Is.False);
+      Assert.That(reader.NativeMetadataStatus, Is.EqualTo("journal-invalid"));
+    });
+  }
+
   [Test, Category("Spec")]
   public void ExtentMap_ReservesUndecodedVolumeAndMarksOnlyExternalTailFree() {
     const long declaredLength = 3L << 12;

Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Tux3/Tux3FormatDescriptor.cs

@@ -5,19 +5,17 @@
 namespace FileSystem.Tux3;
 
 /// <summary>
-/// Native-superblock descriptor for the linux-tux3 research filesystem.
+/// Native metadata descriptor for the linux-tux3 research filesystem.
 /// </summary>
 /// <remarks>
-/// The descriptor recognises real linux-tux3 disk-format revisions and parses the packed,
-/// big-endian <c>struct disksuper</c> at byte 4096. Native tree traversal and mutation are not
-/// implemented, so Create/Modify/Defragment capabilities are intentionally withheld rather than
-/// routing files through a private side-table that no TUX3 implementation understands.
+/// The descriptor recognises native linux-tux3 disk-format revisions, parses the packed big-endian
+/// superblock, and uses the native inode tree, allocation-bitmap data tree, and replayable allocation
+/// log records to prove free/allocated block runs. Active structural tree-log records deliberately
+/// disable the in-volume allocation map until structural replay is implemented.
 ///
-/// <para>The on-disk <c>volblocks</c> field does, however, define the native volume boundary.
-/// Bytes after that boundary are outside the TUX3 volume and can therefore be reported as free,
-/// wiped, and removed by shrink without interpreting or modifying any native tree. Everything
-/// inside the declared volume remains fail-closed as metadata-reserved until allocation-tree
-/// traversal exists.</para>
+/// <para>Create/Modify/Defragment capabilities remain withheld. Allocation-tree parsing is enough
+/// to make layout and Wipe useful inside the declared volume, but moving or deleting native objects
+/// additionally requires directory/inode traversal and transactional metadata/log writing.</para>
 /// </remarks>
 public sealed class Tux3FormatDescriptor :
   IFormatDescriptor, IArchiveFormatOperations, ISyntheticEntryNames, IFilesystemExtentMap, IArchiveShrinkable {
@@ -67,8 +65,8 @@ public sealed class Tux3FormatDescriptor :
 
   /// <inheritdoc />
   public string Description =>
-    "TUX3 version-tree research filesystem — native big-endian superblock detection/metadata, " +
-    "fail-closed layout mapping, external-padding wipe/shrink; native tree traversal and writing not yet implemented.";
+    "TUX3 version-tree research filesystem — native big-endian superblock, allocation-tree and journal parsing; " +
+    "fail-closed in-volume free-space layout/wipe plus external-tail shrink; native mutation not yet implemented.";
 
   /// <inheritdoc />
   public List<ArchiveEntryInfo> List(Stream stream, string? password) {
@@ -92,9 +90,11 @@ public void Extract(Stream stream, string outputDir, string? password, string[]?
   }
 
   /// <summary>
-  /// Enumerates the provable byte layout without guessing at undecoded TUX3 allocation state.
-  /// The declared native volume is reserved wholesale; only trailing bytes outside it are free.
-  /// Invalid or arithmetically unrepresentable volume metadata reserves the complete physical image.
+  /// Enumerates only byte ranges proven by native metadata. When the allocation bitmap and active
+  /// allocation-only journal records are trustworthy, allocated runs stay metadata-reserved and
+  /// unallocated runs are exposed as Free. Otherwise the declared volume remains reserved wholesale.
+  /// Bytes physically appended beyond the declared TUX3 volume are always outside the filesystem.
+  /// Invalid, unrepresentable, or truncated volume metadata reserves the complete physical image.
   /// </summary>
   public IEnumerable<DefragBlockInfo> EnumerateExtents(Stream image) {
     ArgumentNullException.ThrowIfNull(image);
@@ -107,13 +107,49 @@ public IEnumerable<DefragBlockInfo> EnumerateExtents(Stream image) {
       if (!TryGetDeclaredVolumeLength(reader, out var volumeLength))
         return ReserveWholeImage(imageLength);
 
-      if (volumeLength >= imageLength)
+      // A volume declaring more blocks than the image physically holds is truncated. Its allocation
+      // map describes blocks that are not there, so nothing inside it is provable and the complete
+      // physical image stays reserved. An exactly-sized volume is healthy and keeps the map path.
+      if (volumeLength > imageLength)
         return ReserveWholeImage(imageLength);
 
-      return [
-        new DefragBlockInfo(0, volumeLength, DefragBlockKind.MetadataReserved, "TUX3 volume (allocation map unresolved)"),
-        new DefragBlockInfo(volumeLength, imageLength - volumeLength, DefragBlockKind.Free, "Trailing bytes outside TUX3 volume"),
-      ];
+      var extents = new List<DefragBlockInfo>();
+      if (reader.AllocationMapValid && reader.AllocationRuns.Count > 0) {
+        var blockSize = 1UL << reader.BlockBits;
+        foreach (var run in reader.AllocationRuns) {
+          if (run.BlockCount == 0 || run.StartBlock > (ulong)long.MaxValue / blockSize ||
+              run.BlockCount > (ulong)long.MaxValue / blockSize)
+            return ReserveWholeImage(imageLength);
+
+          var offset = run.StartBlock * blockSize;
+          var length = run.BlockCount * blockSize;
+          if (offset > (ulong)volumeLength || length > (ulong)volumeLength - offset)
+            return ReserveWholeImage(imageLength);
+
+          extents.Add(new DefragBlockInfo(
+            (long)offset,
+            (long)length,
+            run.IsAllocated ? DefragBlockKind.MetadataReserved : DefragBlockKind.Free,
+            run.IsAllocated
+              ? "TUX3 allocated blocks (ownership unresolved)"
+              : "TUX3 free blocks (bitmap + journal)"));
+        }
+      } else if (volumeLength > 0) {
+        extents.Add(new DefragBlockInfo(
+          0,
+          volumeLength,
+          DefragBlockKind.MetadataReserved,
+          "TUX3 volume (allocation map unresolved)"));
+      }
+
+      if (volumeLength < imageLength)
+        extents.Add(new DefragBlockInfo(
+          volumeLength,
+          imageLength - volumeLength,
+          DefragBlockKind.Free,
+          "Trailing bytes outside TUX3 volume"));
+
+      return extents;
     } catch (InvalidDataException) {
       return ReserveWholeImage(imageLength);
     } catch (IOException) {