Close libarchive archive/filter parity gaps

Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs

@@ -322,7 +322,7 @@ private static void ConvertAndVerify(Pair pair, IFormatDescriptor srcDesc,
       }
 
       var expected = ExpectedFiles(srcDesc, dstDesc);
-      VerifyPayload(pair, dstPath, dstEntries, expected);
+      VerifyPayload(pair, dstDesc!, dstPath, dstEntries, expected);
   }
 
   /// <summary>
@@ -355,11 +355,18 @@ private static Dictionary<string, byte[]> ExpectedFiles(IFormatDescriptor src, I
   /// case-folding targets per the documented domain quirks; content is always
   /// authoritative.
   /// </summary>
-  private static void VerifyPayload(Pair pair, string dstPath,
+  private static void VerifyPayload(Pair pair, IFormatDescriptor dstDesc, string dstPath,
       List<ArchiveEntry> dstEntries, Dictionary<string, byte[]> expected) {
 
     var nameSynth = NameSynthesizingTargets.Contains(pair.TargetId);
 
+    // A target that advertises creation but not extraction records metadata
+    // rather than bytes — mtree is a filesystem manifest, and the format has
+    // nowhere to put a file body. Its entry names are still verifiable, so the
+    // conversion is exercised and only the byte comparison is skipped; the
+    // matrix must not read the missing payload as a conversion failure.
+    var manifestOnly = (dstDesc.Capabilities & FormatCapabilities.CanExtract) == 0;
+
     // Count: the target must carry at least as many files as we expect, unless
     // it is a name-synthesizing single-stream-ish format (then assert >= 1).
     if (nameSynth)
@@ -399,6 +406,9 @@ private static void VerifyPayload(Pair pair, string dstPath,
         $"{pair}: expected file '{name}' missing from target " +
         $"([{string.Join(",", dstEntries.Select(e => e.Name))}]).");
 
+      if (manifestOnly)
+        continue;
+
       var actual = SafeExtract(dstPath, entry!.Name);
       Assert.That(actual, Is.Not.Null, $"{pair}: extraction of '{entry.Name}' returned null.");
       Assert.That(actual, Is.EqualTo(data),

Compression.Tests/Mtree/MtreeLibarchiveInteropTests.cs

@@ -0,0 +1,69 @@
+using Compression.Registry;
+using FileFormat.Mtree;
+
+namespace Compression.Tests.Mtree;
+
+[TestFixture]
+[Category("ArchiveExternalInterop")]
+public class MtreeLibarchiveInteropTests {
+  private string _tmpDir = null!;
+
+  [SetUp]
+  public void Setup() {
+    this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_mtree_libarchive_{Guid.NewGuid():N}");
+    Directory.CreateDirectory(this._tmpDir);
+  }
+
+  [TearDown]
+  public void Teardown() {
+    try { Directory.Delete(this._tmpDir, true); } catch { /* best effort */ }
+  }
+
+  [Test]
+  public void CwbWriter_ManifestListsWithLibarchive() {
+    RequireBsdtar();
+
+    var manifestPath = Path.Combine(this._tmpDir, "cwb.mtree");
+    using (var target = File.Create(manifestPath)) {
+      new MtreeFormatDescriptor().Create(
+        target,
+        [ArchiveInputInfo.InMemory("payload.txt", "mtree-libarchive"u8)],
+        new FormatCreateOptions());
+    }
+
+    var result = FsInteropToolbox.RunWsl($"bsdtar -tf {FsInteropToolbox.WinToWsl(manifestPath)}");
+    Assert.That(result.ExitCode, Is.EqualTo(0),
+      $"libarchive rejected the CWB mtree manifest:\nstdout:\n{result.StdOut}\nstderr:\n{result.StdErr}");
+    Assert.That(result.StdOut, Does.Contain("payload.txt"));
+  }
+
+  [Test]
+  public void LibarchiveWriter_ManifestReadsWithCwb() {
+    RequireBsdtar();
+
+    var inputPath = Path.Combine(this._tmpDir, "payload.txt");
+    var manifestPath = Path.Combine(this._tmpDir, "libarchive.mtree");
+    File.WriteAllBytes(inputPath, "mtree-libarchive"u8.ToArray());
+
+    var result = FsInteropToolbox.RunWsl(
+      $"bsdtar --format=mtree -cf {FsInteropToolbox.WinToWsl(manifestPath)} " +
+      $"-C {FsInteropToolbox.WinToWsl(this._tmpDir)} payload.txt");
+    Assert.That(result.ExitCode, Is.EqualTo(0),
+      $"libarchive failed to create mtree:\nstdout:\n{result.StdOut}\nstderr:\n{result.StdErr}");
+
+    using var source = File.OpenRead(manifestPath);
+    var listed = new MtreeFormatDescriptor().List(source, password: null);
+    var payload = listed.Single(x => x.Name.EndsWith("payload.txt", StringComparison.Ordinal));
+    Assert.Multiple(() => {
+      Assert.That(payload.IsDirectory, Is.False);
+      Assert.That(payload.OriginalSize, Is.EqualTo(new FileInfo(inputPath).Length));
+    });
+  }
+
+  private static void RequireBsdtar() {
+    if (!FsInteropToolbox.WslAvailable)
+      Assert.Ignore("WSL/Linux shell unavailable; install WSL to run libarchive interoperability tests.");
+    if (!FsInteropToolbox.WslHasTool("bsdtar"))
+      Assert.Ignore("'bsdtar' unavailable; install libarchive-tools in the WSL/Linux environment.");
+  }
+}

Compression.Tests/Mtree/MtreeTests.cs

@@ -0,0 +1,115 @@
+using System.Text;
+using Compression.Registry;
+using FileFormat.Mtree;
+
+namespace Compression.Tests.Mtree;
+
+[TestFixture]
+public class MtreeTests {
+  [Test]
+  [Category("RoundTrip")]
+  public void WriterReader_RoundTripsMetadataAndEscapedNames() {
+    using var stream = new MemoryStream();
+    using (var writer = new MtreeWriter(stream, leaveOpen: true)) {
+      writer.WriteEntry(new MtreeEntry {
+        Path = "dir/file name-ä.txt",
+        Type = MtreeEntryType.File,
+        Mode = 0x1A4,
+        Uid = 1000,
+        Gid = 100,
+        Size = 123,
+        LinkTarget = null,
+      });
+      writer.Flush();
+    }
+
+    stream.Position = 0;
+    var entry = new MtreeReader(stream).ReadAll().Single();
+
+    Assert.Multiple(() => {
+      Assert.That(entry.Path, Is.EqualTo("dir/file name-ä.txt"));
+      Assert.That(entry.Type, Is.EqualTo(MtreeEntryType.File));
+      Assert.That(entry.Mode, Is.EqualTo(0x1A4));
+      Assert.That(entry.Uid, Is.EqualTo(1000));
+      Assert.That(entry.Gid, Is.EqualTo(100));
+      Assert.That(entry.Size, Is.EqualTo(123));
+    });
+  }
+
+  [Test]
+  [Category("Compatibility")]
+  public void Reader_AppliesSetUnsetAndClassicDirectoryTraversal() {
+    const string manifest = """
+      #mtree
+      /set type=file uid=1000 gid=100 mode=0644
+      dir type=dir mode=0755
+      file\040one size=3
+      sub type=dir
+      child size=4
+      ..
+      ..
+      /unset uid gid
+      ./link type=link link=target\040name
+      """;
+
+    using var stream = new MemoryStream(Encoding.UTF8.GetBytes(manifest));
+    var entries = new MtreeReader(stream).ReadAll();
+
+    Assert.That(entries.Select(x => x.Path), Is.EqualTo(new[] {
+      "dir",
+      "dir/file one",
+      "dir/sub",
+      "dir/sub/child",
+      "link",
+    }));
+    Assert.Multiple(() => {
+      Assert.That(entries[1].Uid, Is.EqualTo(1000));
+      Assert.That(entries[1].Mode, Is.EqualTo(0x1A4));
+      Assert.That(entries[3].Size, Is.EqualTo(4));
+      Assert.That(entries[4].Type, Is.EqualTo(MtreeEntryType.Link));
+      Assert.That(entries[4].LinkTarget, Is.EqualTo("target name"));
+      Assert.That(entries[4].Uid, Is.Null);
+    });
+  }
+
+  [Test]
+  [Category("Registry")]
+  public void Descriptor_CreateProducesListableManifestWithoutPretendingToContainBodies() {
+    var descriptor = new MtreeFormatDescriptor();
+    using var stream = new MemoryStream();
+    descriptor.Create(
+      stream,
+      [
+        new ArchiveInputInfo("dir", "dir", IsDirectory: true),
+        ArchiveInputInfo.InMemory("dir/payload.bin", [1, 2, 3, 4]),
+      ],
+      new FormatCreateOptions());
+
+    var bytes = stream.ToArray();
+    Assert.That(Encoding.UTF8.GetString(bytes), Does.StartWith("#mtree\n"));
+
+    stream.Position = 0;
+    var listed = descriptor.List(stream, password: null);
+    Assert.That(listed, Has.Count.EqualTo(2));
+    Assert.Multiple(() => {
+      Assert.That(listed[0].Name, Is.EqualTo("dir"));
+      Assert.That(listed[0].IsDirectory, Is.True);
+      Assert.That(listed[1].Name, Is.EqualTo("dir/payload.bin"));
+      Assert.That(listed[1].OriginalSize, Is.EqualTo(4));
+      Assert.That(descriptor.Capabilities.HasFlag(FormatCapabilities.CanExtract), Is.False);
+    });
+
+    stream.Position = 0;
+    Assert.Throws<NotSupportedException>(() => descriptor.Extract(stream, Path.GetTempPath(), null, null));
+  }
+
+  [Test]
+  [Category("Compatibility")]
+  public void Reader_DecodesUtf8OctalByteEscapes() {
+    // UTF-8 ä is C3 A4; mtree escapes bytes, not Unicode code points.
+    const string manifest = "#mtree\n./caf\\303\\244 type=file size=0\n";
+    using var stream = new MemoryStream(Encoding.ASCII.GetBytes(manifest));
+    var entry = new MtreeReader(stream).ReadAll().Single();
+    Assert.That(entry.Path, Is.EqualTo("cafä"));
+  }
+}

Compression.Tests/UuEncoding/B64EncodingLibarchiveInteropTests.cs

@@ -0,0 +1,84 @@
+using FileFormat.Cpio;
+using FileFormat.UuEncoding;
+
+namespace Compression.Tests.UuEncoding;
+
+/// <summary>
+/// Behavioral-oracle coverage for libarchive's b64encode filter. The wire
+/// format is implemented independently from the documented begin-base64
+/// envelope and published vectors; these tests only verify interoperability.
+/// </summary>
+[TestFixture]
+[Category("ArchiveExternalInterop")]
+public class B64EncodingLibarchiveInteropTests {
+  private static readonly byte[] Payload = "libarchive-b64encode-interoperability"u8.ToArray();
+  private string _tmpDir = null!;
+
+  [SetUp]
+  public void Setup() {
+    this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_b64_libarchive_{Guid.NewGuid():N}");
+    Directory.CreateDirectory(this._tmpDir);
+  }
+
+  [TearDown]
+  public void Teardown() {
+    try { Directory.Delete(this._tmpDir, true); } catch { /* best effort */ }
+  }
+
+  [Test]
+  public void CwbBase64WrappedCpio_IsReadByBsdtar() {
+    RequireTool("bsdtar");
+
+    using var cpio = new MemoryStream();
+    using (var writer = new CpioWriter(cpio, CpioArchiveFormat.PortableAscii, leaveOpen: true)) {
+      writer.AddFile("payload.txt", Payload);
+      writer.Finish();
+    }
+    cpio.Position = 0;
+
+    var wrappedPath = Path.Combine(this._tmpDir, "cwb.cpio.b64");
+    using (var output = File.Create(wrappedPath))
+      UuEncoder.EncodeBase64(cpio, output, "payload.cpio");
+
+    var result = FsInteropToolbox.RunWsl($"bsdtar -tf {FsInteropToolbox.WinToWsl(wrappedPath)}");
+    Assert.That(result.ExitCode, Is.EqualTo(0),
+      $"bsdtar rejected CWB b64encode output:\nstdout:\n{result.StdOut}\nstderr:\n{result.StdErr}");
+    Assert.That(result.StdOut, Does.Contain("payload.txt"));
+  }
+
+  [Test]
+  public void LibarchiveBase64WrappedCpio_IsDecodedByCwb() {
+    RequireTool("bsdcpio");
+
+    var inputPath = Path.Combine(this._tmpDir, "payload.txt");
+    var wrappedPath = Path.Combine(this._tmpDir, "libarchive.cpio.b64");
+    File.WriteAllBytes(inputPath, Payload);
+
+    var wslDir = FsInteropToolbox.WinToWsl(this._tmpDir);
+    var wslWrapped = FsInteropToolbox.WinToWsl(wrappedPath);
+    var result = FsInteropToolbox.RunWsl(
+      $"cd {wslDir} && printf 'payload.txt\\n' | bsdcpio -o --format=odc --b64encode > {wslWrapped}");
+    Assert.That(result.ExitCode, Is.EqualTo(0),
+      $"bsdcpio failed to create b64encoded cpio:\nstdout:\n{result.StdOut}\nstderr:\n{result.StdErr}");
+
+    using var wrapped = File.OpenRead(wrappedPath);
+    using var decoded = new MemoryStream();
+    new B64EncodingFormatDescriptor().Decompress(wrapped, decoded);
+    decoded.Position = 0;
+
+    using var reader = new CpioReader(decoded, leaveOpen: true);
+    var entries = reader.ReadAll();
+    Assert.That(entries, Has.Count.EqualTo(1));
+    Assert.Multiple(() => {
+      Assert.That(entries[0].Entry.Name, Is.EqualTo("payload.txt"));
+      Assert.That(entries[0].Data, Is.EqualTo(Payload));
+    });
+  }
+
+  private static void RequireTool(string tool) {
+    if (!FsInteropToolbox.WslAvailable)
+      Assert.Ignore("WSL/Linux shell unavailable; install WSL to run libarchive interoperability tests.");
+    if (!FsInteropToolbox.WslHasTool(tool))
+      Assert.Ignore($"'{tool}' unavailable; install libarchive-tools in the WSL/Linux environment.");
+  }
+}