+ PyInstaller: create CArchive (PKG) payloads, validated by PyInstaller's own reader and writer

Compression.Lib/FormatDetector.cs

@@ -712,9 +712,37 @@ public static Format Detect(string path) {
         return overlay;
     }
 
+    if (byMagic == Format.Unknown && IsBarePyInstallerArchive(path))
+      return Format.PyInstaller;
+
     return byMagic;
   }
 
+  /// <summary>
+  /// A bare PyInstaller CArchive (the PKG PyInstaller builds before appending it to a
+  /// bootloader) has no leading magic: it ends in the 88-byte cookie, whose package length
+  /// then covers the whole file and whose TOC ends exactly where the cookie starts.
+  /// </summary>
+  private static bool IsBarePyInstallerArchive(string path) {
+    const int cookieLength = 88;
+    try {
+      using var fs = File.OpenRead(path);
+      if (fs.Length < cookieLength || fs.Length > uint.MaxValue) return false;
+      Span<byte> cookie = stackalloc byte[cookieLength];
+      fs.Seek(-cookieLength, SeekOrigin.End);
+      fs.ReadExactly(cookie);
+      if (!cookie[..8].SequenceEqual(PyInstallerCookie)) return false;
+      var packageLength = System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(cookie[8..]);
+      var tocOffset = System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(cookie[12..]);
+      var tocLength = System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(cookie[16..]);
+      return packageLength == fs.Length && (long)tocOffset + tocLength == fs.Length - cookieLength;
+    } catch (IOException) {
+      return false;
+    } catch (UnauthorizedAccessException) {
+      return false;
+    }
+  }
+
   /// <summary>
   /// Picks the claimant of a shared extension whose own magic matches the file,
   /// preferring the most confident. Returns <see cref="Format.Unknown" /> when

Compression.Tests/PyInstaller/PyInstallerCArchiveOracleTests.cs

@@ -0,0 +1,156 @@
+using Win32Exception = System.ComponentModel.Win32Exception;
+using System.Diagnostics;
+using System.Text;
+using Compression.Lib;
+using Compression.Registry;
+using FileFormat.PyInstaller;
+
+namespace Compression.Tests.PyInstaller;
+
+/// <summary>
+/// Our CArchive (PKG) writer judged by PyInstaller itself: its own
+/// <c>PyInstaller.archive.readers.CArchiveReader</c> must list and extract what we wrote, and
+/// for stored entries its own <c>CArchiveWriter</c> must produce the very same bytes. Compressed
+/// entries are compared by content only: CPython's zlib and .NET's deflate make different
+/// (equally valid) streams. The tests run wherever a Python with PyInstaller installed is on the
+/// PATH and are ignored elsewhere.
+/// </summary>
+[TestFixture]
+[Category("ArchiveExternal")]
+public sealed class PyInstallerCArchiveOracleTests {
+
+  private string _tmpDir = null!;
+
+  [SetUp]
+  public void SetUp() {
+    this._tmpDir = Path.Combine(Path.GetTempPath(), "cwb_pyi_" + Guid.NewGuid().ToString("N")[..8]);
+    Directory.CreateDirectory(this._tmpDir);
+  }
+
+  [TearDown]
+  public void TearDown() {
+    try { Directory.Delete(this._tmpDir, recursive: true); } catch { /* best effort */ }
+  }
+
+  private static readonly Lazy<string?> Python = new(FindPythonWithPyInstaller);
+
+  private static string? FindPythonWithPyInstaller() {
+    foreach (var candidate in new[] { "python3", "python" }) {
+      try {
+        var (code, stdout, _) = Run(candidate, ["-c", "import PyInstaller.archive.readers, PyInstaller.archive.writers, sys; print(sys.version_info[0] * 100 + sys.version_info[1])"]);
+        if (code == 0 && int.TryParse(stdout.Trim(), out _)) return candidate;
+      } catch (Win32Exception) {
+        // not on PATH
+      }
+    }
+    return null;
+  }
+
+  private static string RequirePython() {
+    var python = Python.Value;
+    if (python is null) Assert.Ignore("No Python with PyInstaller on the PATH (`pip install pyinstaller`).");
+    return python!;
+  }
+
+  private static (int ExitCode, string StdOut, string StdErr) Run(string file, IEnumerable<string> arguments) {
+    var start = new ProcessStartInfo(file) {
+      RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true,
+    };
+    foreach (var argument in arguments) start.ArgumentList.Add(argument);
+    using var process = Process.Start(start)!;
+    var stdout = process.StandardOutput.ReadToEndAsync();
+    var stderr = process.StandardError.ReadToEnd();
+    process.WaitForExit();
+    return (process.ExitCode, stdout.Result, stderr);
+  }
+
+  private static int PythonVersion(string python) => int.Parse(Run(python, ["-c", "import sys; print(sys.version_info[0] * 100 + sys.version_info[1])"]).StdOut.Trim());
+
+  /// <summary>Lists name, typecode, compression flag and the SHA-256 of each extracted entry.</summary>
+  private const string ReaderScript = """
+    import hashlib, sys
+    from PyInstaller.archive.readers import CArchiveReader
+    archive = CArchiveReader(sys.argv[1])
+    for name in sorted(archive.toc):
+        offset, length, uncompressed, flag, typecode = archive.toc[name]
+        data = archive.extract(name)
+        assert len(data) == uncompressed, name
+        print(f"{name}\t{typecode}\t{flag}\t{hashlib.sha256(data).hexdigest()}")
+    """;
+
+  private const string WriterScript = """
+    import sys
+    from PyInstaller.archive.writers import CArchiveWriter
+    out, libname, typecode = sys.argv[1], sys.argv[2], sys.argv[3]
+    names = sys.argv[4::2]
+    sources = sys.argv[5::2]
+    CArchiveWriter(out, [(n, s, False, typecode) for n, s in zip(names, sources)], libname)
+    """;
+
+  private static readonly (string Name, byte[] Data)[] Payload = [
+    ("readme.txt", Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("PyInstaller CArchive payload ", 64)))),
+    ("empty.bin", []),
+    // 18-byte entry header + 13 name bytes + NUL = 32: already aligned, so no padding.
+    ("exactly13.bin", [1, 2, 3]),
+    ("assets/nested/deeper.dat", Enumerable.Range(0, 70_000).Select(static i => (byte)(i * 31)).ToArray()),
+  ];
+
+  private byte[] WriteOurs(string method, int version, string libraryName, string targetOs = "posix") {
+    using var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output,
+      [.. Payload.Select(static p => ArchiveInputInfo.InMemory(p.Name, p.Data))],
+      new FormatCreateOptions(method) {
+        FormatSpecific = new(StringComparer.OrdinalIgnoreCase) {
+          ["PythonVersion"] = version.ToString(System.Globalization.CultureInfo.InvariantCulture),
+          ["PythonLibraryName"] = libraryName,
+          ["TypeCode"] = "x",
+          ["TargetOs"] = targetOs,
+        },
+      });
+    return output.ToArray();
+  }
+
+  [TestCase("zlib", 1, TestName = "GivenZlibEntries_WhenPyInstallersReaderOpensOurPkg_ThenEveryEntryExtractsIdentically")]
+  [TestCase("stored", 0, TestName = "GivenStoredEntries_WhenPyInstallersReaderOpensOurPkg_ThenEveryEntryExtractsIdentically")]
+  public void PyInstallerReader_ExtractsWhatWeWrote(string method, int expectedFlag) {
+    var python = RequirePython();
+    var path = Path.Combine(this._tmpDir, "ours.pkg");
+    File.WriteAllBytes(path, this.WriteOurs(method, PythonVersion(python), "libpython3.13.so.1.0"));
+
+    var (code, stdout, stderr) = Run(python, ["-c", ReaderScript, path]);
+    Assert.That(code, Is.EqualTo(0), $"PyInstaller's CArchiveReader rejected our PKG:\n{stderr}");
+
+    var expected = Payload
+      .OrderBy(static p => p.Name, StringComparer.Ordinal)
+      .Select(p => $"{p.Name}\tx\t{expectedFlag}\t{Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(p.Data))}");
+    Assert.That(stdout.ReplaceLineEndings("\n").TrimEnd('\n').Split('\n'), Is.EqualTo(expected));
+  }
+
+  [Test, Category("HappyPath")]
+  public void GivenStoredEntries_WhenPyInstallersOwnWriterBuildsTheSamePkg_ThenTheBytesAreIdentical() {
+    var python = RequirePython();
+    var version = PythonVersion(python);
+    const string libraryName = "python313.dll";
+    var arguments = new List<string> { "-c", WriterScript, Path.Combine(this._tmpDir, "reference.pkg"), libraryName, "x" };
+    foreach (var (name, data) in Payload) {
+      var source = Path.Combine(this._tmpDir, "src_" + arguments.Count);
+      File.WriteAllBytes(source, data);
+      arguments.Add(name);
+      arguments.Add(source);
+    }
+
+    var (code, _, stderr) = Run(python, arguments);
+    Assert.That(code, Is.EqualTo(0), $"PyInstaller's CArchiveWriter failed:\n{stderr}");
+
+    var reference = File.ReadAllBytes(Path.Combine(this._tmpDir, "reference.pkg"));
+    Assert.That(this.WriteOurs("stored", version, libraryName, OperatingSystem.IsWindows() ? "windows" : "posix"), Is.EqualTo(reference),
+      "a stored PKG must be byte-identical to the one PyInstaller writes for the same entries");
+  }
+
+  [Test, Category("HappyPath")]
+  public void GivenABarePkgWithoutAnExtension_WhenDetecting_ThenTheTrailingCookieIdentifiesPyInstaller() {
+    var path = Path.Combine(this._tmpDir, "PKG-00");
+    File.WriteAllBytes(path, this.WriteOurs("zlib", 313, "python313.dll"));
+    Assert.That(FormatDetector.Detect(path), Is.EqualTo(FormatDetector.Format.PyInstaller));
+  }
+}

Compression.Tests/PyInstaller/PyInstallerTests.cs

@@ -2,6 +2,7 @@
 using System.IO.Compression;
 using System.Text;
 using Compression.Lib;
+using Compression.Registry;
 using FileFormat.PyInstaller;
 
 namespace Compression.Tests.PyInstaller;
@@ -276,6 +277,161 @@ public void Descriptor_Extract_WritesDecompressedEntries() {
     }
   }
 
+  [Category("HappyPath")]
+  [TestCase(0)]
+  [TestCase(1)]
+  [TestCase(5)]
+  [TestCase(9)]
+  public void Create_ZlibRoundTripsAtSupportedLevels(int level) {
+    var payload = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("PyInstaller CArchive payload ", 64)));
+    var output = new MemoryStream();
+    var descriptor = new PyInstallerFormatDescriptor();
+    descriptor.Create(output, [ArchiveInputInfo.InMemory("assets/readme.txt", payload)], new Compression.Registry.FormatCreateOptions("zlib") {
+      Level = level,
+      FormatSpecific = new(StringComparer.OrdinalIgnoreCase) {
+        ["PythonVersion"] = "312",
+        ["PythonLibraryName"] = "libpython3.12.so",
+        ["TypeCode"] = "b",
+      },
+    });
+
+    output.Position = 0;
+    var reader = new PyInstallerReader(output);
+    var entry = reader.ReadToc().Single();
+    Assert.Multiple(() => {
+      Assert.That(reader.PythonVersion, Is.EqualTo(312));
+      Assert.That(reader.PythonLibraryName, Is.EqualTo("libpython3.12.so"));
+      Assert.That(entry.Name, Is.EqualTo("assets/readme.txt"));
+      Assert.That(entry.TypeCode, Is.EqualTo('b'));
+      Assert.That(entry.IsCompressed, Is.True);
+      Assert.That(reader.GetData(entry), Is.EqualTo(payload));
+    });
+  }
+
+  [Category("HappyPath")]
+  [Test]
+  public void Create_StoredRoundTripsAndSetsTypeCode() {
+    var payload = new byte[] { 0, 1, 2, 3, 255 };
+    var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory("payload.bin", payload)], new Compression.Registry.FormatCreateOptions("stored") {
+      FormatSpecific = new(StringComparer.OrdinalIgnoreCase) { ["TypeCode"] = "x" },
+    });
+
+    output.Position = 0;
+    var reader = new PyInstallerReader(output);
+    var entry = reader.ReadToc().Single();
+    Assert.Multiple(() => {
+      Assert.That(entry.IsCompressed, Is.False);
+      Assert.That(entry.TypeCode, Is.EqualTo('x'));
+      Assert.That(reader.GetData(entry), Is.EqualTo(payload));
+    });
+  }
+
+  [TestCase("o", TestName = "GivenTheOptionTypeCode_WhenCreating_ThenItIsRefused")]
+  [TestCase("s", TestName = "GivenTheScriptTypeCode_WhenCreating_ThenItIsRefused")]
+  [TestCase("z", TestName = "GivenThePyzTypeCode_WhenCreating_ThenItIsRefused")]
+  [TestCase("", TestName = "GivenAnEmptyTypeCode_WhenCreating_ThenItIsRefused")]
+  [Category("Exceptional")]
+  public void Create_RefusesTypeCodesThatDoNotMeanRawBytes(string typeCode)
+    => Assert.Throws<ArgumentException>(() => new PyInstallerFormatDescriptor().Create(new MemoryStream(),
+      [ArchiveInputInfo.InMemory("a.bin", new byte[] { 1 })],
+      new Compression.Registry.FormatCreateOptions("stored") { FormatSpecific = new(StringComparer.OrdinalIgnoreCase) { ["TypeCode"] = typeCode } }));
+
+  [Category("BoundaryCase")]
+  [TestCase(63, true, TestName = "GivenA63ByteLibraryName_WhenCreating_ThenItFitsTheCookieWithItsTerminator")]
+  [TestCase(64, false, TestName = "GivenA64ByteLibraryName_WhenCreating_ThenItIsRefusedForLackOfATerminator")]
+  public void Create_LibraryNameLengthBoundary(int length, bool accepted) {
+    var options = new Compression.Registry.FormatCreateOptions("stored") {
+      FormatSpecific = new(StringComparer.OrdinalIgnoreCase) { ["PythonLibraryName"] = new string('l', length) },
+    };
+    Action create = () => new PyInstallerFormatDescriptor().Create(new MemoryStream(), [ArchiveInputInfo.InMemory("a.bin", new byte[] { 1 })], options);
+    if (accepted) Assert.DoesNotThrow(create);
+    else Assert.Throws<ArgumentException>(create);
+  }
+
+  [Category("EquivalenceClass")]
+  [Test]
+  public void GivenNoLibraryName_WhenCreating_ThenTheCookieNamesTheVersionsWindowsLibrary() {
+    var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory("a.bin", new byte[] { 1 })],
+      new Compression.Registry.FormatCreateOptions("stored") { FormatSpecific = new(StringComparer.OrdinalIgnoreCase) { ["PythonVersion"] = "312" } });
+    output.Position = 0;
+    Assert.That(new PyInstallerReader(output).PythonLibraryName, Is.EqualTo("python312.dll"));
+  }
+
+  [Category("BoundaryCase")]
+  [TestCase("a", TestName = "GivenAOneByteName_WhenWritingTheToc_ThenTheEntryIsPaddedTo32")]
+  [TestCase("exactly13.bin", TestName = "GivenANameThatFillsTheEntry_WhenWritingTheToc_ThenNoPaddingIsAdded")]
+  [TestCase("fourteen-b.bin", TestName = "GivenANameOneByteTooLong_WhenWritingTheToc_ThenTheEntryGrowsTo48")]
+  public void Create_AlignsEveryTocEntryTo16Bytes(string name) {
+    var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory(name, new byte[] { 7 })],
+      new Compression.Registry.FormatCreateOptions("stored"));
+    var bytes = output.ToArray();
+    var cookie = bytes.AsSpan(bytes.Length - 88);
+    var tocOffset = (int)System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(cookie[12..]);
+    var tocLength = (int)System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(cookie[16..]);
+    var entryLength = (int)System.Buffers.Binary.BinaryPrimitives.ReadUInt32BigEndian(bytes.AsSpan(tocOffset));
+    Assert.Multiple(() => {
+      Assert.That(entryLength % 16, Is.Zero);
+      Assert.That(entryLength, Is.EqualTo(tocLength));
+      Assert.That(entryLength, Is.EqualTo((18 + name.Length + 1 + 15) / 16 * 16));
+    });
+  }
+
+  [Category("EquivalenceClass")]
+  [TestCase("posix", "lib/data.bin", TestName = "GivenAPosixTarget_WhenCreating_ThenNestedNamesUseForwardSlashes")]
+  [TestCase("windows", "lib\\data.bin", TestName = "GivenAWindowsTarget_WhenCreating_ThenNestedNamesUseBackslashes")]
+  public void Create_UsesTheTargetBootloadersSeparator(string targetOs, string expectedName) {
+    var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory("lib/data.bin", new byte[] { 1 })],
+      new Compression.Registry.FormatCreateOptions("stored") { FormatSpecific = new(StringComparer.OrdinalIgnoreCase) { ["TargetOs"] = targetOs } });
+    output.Position = 0;
+    Assert.That(new PyInstallerReader(output).ReadToc().Single().Name, Is.EqualTo(expectedName));
+  }
+
+  [Category("HappyPath")]
+  [Test]
+  public void GivenAStreamThatAlreadyHoldsABootloader_WhenCreating_ThenThePkgIsAppendedWithRelativeOffsets() {
+    var stub = Enumerable.Repeat((byte)0xCC, 5000).ToArray();
+    var payload = Encoding.ASCII.GetBytes("appended after the stub");
+    var output = new MemoryStream();
+    output.Write(stub);
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory("a.txt", payload)],
+      new Compression.Registry.FormatCreateOptions("zlib"));
+
+    Assert.That(output.ToArray().AsSpan(0, stub.Length).SequenceEqual(stub), Is.True, "the stub must be left alone");
+    output.Position = 0;
+    var reader = new PyInstallerReader(output);
+    Assert.That(reader.GetData(reader.ReadToc().Single()), Is.EqualTo(payload));
+  }
+
+  [Category("BoundaryCase")]
+  [Test]
+  public void GivenAnEmptyFile_WhenCompressing_ThenTheEntryIsAValidZlibStream() {
+    var output = new MemoryStream();
+    new PyInstallerFormatDescriptor().Create(output, [ArchiveInputInfo.InMemory("empty.bin", [])],
+      new Compression.Registry.FormatCreateOptions("zlib"));
+    output.Position = 0;
+    var reader = new PyInstallerReader(output);
+    var entry = reader.ReadToc().Single();
+    Assert.Multiple(() => {
+      Assert.That(entry.IsCompressed, Is.True);
+      Assert.That(entry.CompressedLength, Is.GreaterThan(0u), "zero bytes is not a zlib stream");
+      Assert.That(reader.GetData(entry), Is.Empty);
+    });
+  }
+
+  [Category("Exceptional")]
+  [Test]
+  public void Create_RejectsUnsafeEntryNames() {
+    var output = new MemoryStream();
+    Assert.Throws<ArgumentException>(() => new PyInstallerFormatDescriptor().Create(
+      output,
+      [ArchiveInputInfo.InMemory("../escape.txt", new byte[] { 1, 2, 3 })],
+      new Compression.Registry.FormatCreateOptions("stored")));
+  }
+
   [Category("Exceptional")]
   [Test]
   public void Constructor_WithoutCookie_Throws() {