Skip to content

Add native span paths for JPEG XL #394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hawkynt merged 2 commits into from
Sep 28, 2026
+187 −35
Merged

Add native span paths for JPEG XL #394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
07e423a
Add native JPEG XL span layout
Hawkynt Sep 25, 2026
3858f45
Cover JPEG XL native span paths
Hawkynt Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 142 additions & 34 deletions Compression.Lib/FileFormats/FileFormat.Jxl/JxlFormatDescriptor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,83 +114,191 @@ public void Extract(Stream stream, string outputDir, string? password, string[]?
}
}

private readonly record struct SourceRange(int Offset, int Length);

private sealed record EntryLayout(
string Name,
IReadOnlyList<SourceRange> Ranges,
byte[]? Generated = null) {

public int Size {
get {
if (this.Generated is { } generated)
return generated.Length;

var total = 0;
foreach (var range in this.Ranges)
total = checked(total + range.Length);
return total;
}
}
}

private static List<(string Name, byte[] Data)> BuildEntries(Stream stream) {
using var ms = new MemoryStream();
stream.CopyTo(ms);
var blob = ms.ToArray();

return BuildLayout(blob)
.Select(entry => (entry.Name, Materialize(blob, entry)))
.ToList();
}

List<ArchiveEntryInfo> IArchiveFormatOperations.ListSpan(ReadOnlySpan<byte> archive, string? password) {
List<EntryLayout> entries;
try {
entries = BuildLayout(archive);
} catch {
return [];
}

return entries.Select((entry, index) => new ArchiveEntryInfo(
Index: index, Name: entry.Name,
OriginalSize: entry.Size, CompressedSize: entry.Size,
Method: "stored", IsDirectory: false, IsEncrypted: false, LastModified: null
)).ToList();
}

void IArchiveFormatOperations.ExtractSpan(
ReadOnlySpan<byte> archive, string outputDir, string? password, string[]? files) {
List<EntryLayout> entries;
try {
entries = BuildLayout(archive);
} catch {
return;
}

foreach (var entry in entries) {
if (files is { Length: > 0 } && !MatchesFilter(entry.Name, files))
continue;

if (entry.Generated is { } generated) {
WriteFile(outputDir, entry.Name, generated);
continue;
}

using var output = CreateEntryFile(outputDir, entry.Name);
foreach (var range in entry.Ranges)
output.Write(archive.Slice(range.Offset, range.Length));
}
}

private static List<EntryLayout> BuildLayout(ReadOnlySpan<byte> blob) {
var isNaked = blob.Length >= 2 && blob[0] == 0xFF && blob[1] == 0x0A;
var isBox = blob.Length >= 12
&& blob[0] == 0x00 && blob[1] == 0x00 && blob[2] == 0x00 && blob[3] == 0x0C
&& blob[4] == 0x4A && blob[5] == 0x58 && blob[6] == 0x4C && blob[7] == 0x20
&& blob[8] == 0x0D && blob[9] == 0x0A && blob[10] == 0x87 && blob[11] == 0x0A;

var entries = new List<(string Name, byte[] Data)> {
("FULL.jxl", blob),
var entries = new List<EntryLayout> {
new("FULL.jxl", [new SourceRange(0, blob.Length)]),
};

var meta = new StringBuilder();
meta.AppendLine("; JPEG XL container metadata");

byte[] codestream = [];
byte[]? exif = null, xmp = null, jumb = null;
int? level = null;
var codestreamRanges = new List<SourceRange>();
SourceRange? exif = null;
SourceRange? xmp = null;
SourceRange? jumb = null;

if (isNaked) {
meta.AppendLine("form=naked");
codestream = blob;
codestreamRanges.Add(new SourceRange(0, blob.Length));
} else if (isBox) {
meta.AppendLine("form=box");
var boxes = new BoxParser().Parse(blob);

// jxll: single int8 level (usually 5 or 10).
var jxll = BoxParser.Find(boxes, "jxll");
if (jxll != null && jxll.BodyLength >= 1) {
level = blob[(int)jxll.BodyOffset];
meta.Append("level=").AppendLine(level.Value.ToString(CultureInfo.InvariantCulture));
if (jxll != null && jxll.BodyLength >= 1 && IsValidRange(blob, jxll.BodyOffset, 1)) {
var level = blob[(int)jxll.BodyOffset];
meta.Append("level=").AppendLine(level.ToString(CultureInfo.InvariantCulture));
}

// Concatenate codestream: prefer single jxlc, else ordered jxlp parts.
var jxlc = BoxParser.Find(boxes, "jxlc");
if (jxlc != null && jxlc.BodyLength > 0) {
codestream = blob.AsSpan((int)jxlc.BodyOffset, (int)jxlc.BodyLength).ToArray();
if (jxlc != null && jxlc.BodyLength > 0 && TryRange(blob, jxlc.BodyOffset, jxlc.BodyLength, out var jxlcRange)) {
codestreamRanges.Add(jxlcRange);
} else {
using var csMs = new MemoryStream();
foreach (var part in BoxParser.FindAll(boxes, "jxlp")) {
// Each jxlp body starts with a 4-byte partial index; skip it to get raw codestream bytes.
if (part.BodyLength <= 4) continue;
csMs.Write(blob, (int)(part.BodyOffset + 4), (int)(part.BodyLength - 4));
if (part.BodyLength <= 4)
continue;
if (TryRange(blob, part.BodyOffset + 4, part.BodyLength - 4, out var partRange))
codestreamRanges.Add(partRange);
}
codestream = csMs.ToArray();
}

var exifBox = BoxParser.Find(boxes, "Exif");
if (exifBox != null && exifBox.BodyLength > 0)
exif = blob.AsSpan((int)exifBox.BodyOffset, (int)exifBox.BodyLength).ToArray();
if (exifBox != null && exifBox.BodyLength > 0
&& TryRange(blob, exifBox.BodyOffset, exifBox.BodyLength, out var exifRange))
exif = exifRange;

var xmpBox = BoxParser.Find(boxes, "xml ");
if (xmpBox != null && xmpBox.BodyLength > 0)
xmp = blob.AsSpan((int)xmpBox.BodyOffset, (int)xmpBox.BodyLength).ToArray();
if (xmpBox != null && xmpBox.BodyLength > 0
&& TryRange(blob, xmpBox.BodyOffset, xmpBox.BodyLength, out var xmpRange))
xmp = xmpRange;

var jumbBox = BoxParser.Find(boxes, "jumb");
if (jumbBox != null && jumbBox.BodyLength > 0)
jumb = blob.AsSpan((int)jumbBox.BodyOffset, (int)jumbBox.BodyLength).ToArray();
if (jumbBox != null && jumbBox.BodyLength > 0
&& TryRange(blob, jumbBox.BodyOffset, jumbBox.BodyLength, out var jumbRange))
jumb = jumbRange;
} else {
meta.AppendLine("form=unknown");
}

meta.Append("has_exif=").AppendLine(exif != null ? "true" : "false");
meta.Append("has_xmp=").AppendLine(xmp != null ? "true" : "false");
meta.Append("has_jumb=").AppendLine(jumb != null ? "true" : "false");
meta.Append("codestream_size=").AppendLine(codestream.Length.ToString(CultureInfo.InvariantCulture));
var codestreamSize = 0;
foreach (var range in codestreamRanges)
codestreamSize = checked(codestreamSize + range.Length);

entries.Add(("metadata.ini", Encoding.UTF8.GetBytes(meta.ToString())));
meta.Append("has_exif=").AppendLine(exif.HasValue ? "true" : "false");
meta.Append("has_xmp=").AppendLine(xmp.HasValue ? "true" : "false");
meta.Append("has_jumb=").AppendLine(jumb.HasValue ? "true" : "false");
meta.Append("codestream_size=").AppendLine(codestreamSize.ToString(CultureInfo.InvariantCulture));

if (codestream.Length > 0)
entries.Add(("codestream.jxl", codestream));
entries.Add(new EntryLayout("metadata.ini", [], Encoding.UTF8.GetBytes(meta.ToString())));

if (exif != null) entries.Add(("metadata/exif.bin", exif));
if (xmp != null) entries.Add(("metadata/xmp.xml", xmp));
if (jumb != null) entries.Add(("metadata/jumb.bin", jumb));
if (codestreamRanges.Count > 0)
entries.Add(new EntryLayout("codestream.jxl", codestreamRanges));

if (exif is { } exifRangeValue)
entries.Add(new EntryLayout("metadata/exif.bin", [exifRangeValue]));
if (xmp is { } xmpRangeValue)
entries.Add(new EntryLayout("metadata/xmp.xml", [xmpRangeValue]));
if (jumb is { } jumbRangeValue)
entries.Add(new EntryLayout("metadata/jumb.bin", [jumbRangeValue]));

return entries;
}

private static byte[] Materialize(ReadOnlySpan<byte> blob, EntryLayout entry) {
if (entry.Generated is { } generated)
return generated;

if (entry.Ranges.Count == 1) {
var range = entry.Ranges[0];
return blob.Slice(range.Offset, range.Length).ToArray();
}

var result = new byte[entry.Size];
var destinationOffset = 0;
foreach (var range in entry.Ranges) {
blob.Slice(range.Offset, range.Length).CopyTo(result.AsSpan(destinationOffset));
destinationOffset += range.Length;
}
return result;
}

private static bool IsValidRange(ReadOnlySpan<byte> blob, long offset, long length) =>
offset >= 0 && length >= 0 && offset <= blob.Length && length <= blob.Length - offset;

private static bool TryRange(
ReadOnlySpan<byte> blob, long offset, long length, out SourceRange range) {
if (!IsValidRange(blob, offset, length)) {
range = default;
return false;
}

range = new SourceRange((int)offset, (int)length);
return true;
}

}
46 changes: 45 additions & 1 deletion Compression.Tests/Registry/NativeSpanArchiveInputTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using FileFormat.Pcap;
using FileFormat.Pcapng;
using FileFormat.Jp2;
using FileFormat.Jxl;
using FileFormat.Psb;
using FileFormat.Psd;
using FileFormat.Sup;
Expand Down Expand Up @@ -91,14 +92,17 @@ public void SourceRangeFormats_ExtractSpan_LargePayloads_DoNotCopyWholeArchive()
BuildPsdCase(LargePayloadSize),
BuildPsbCase(LargePayloadSize),
BuildJp2Case(LargePayloadSize),
BuildJxlCase(LargePayloadSize),
}) {
var directory = Path.Combine(Path.GetTempPath(), $"cwb-span-range-{Guid.NewGuid():N}");
try {
var warmDirectory = Path.Combine(directory, "warm");
var warmCase = testCase.Name switch {
"PSD" => BuildPsdCase(257),
"PSB" => BuildPsbCase(257),
_ => BuildJp2Case(257),
"JP2" => BuildJp2Case(257),
"JXL" => BuildJxlCase(257),
_ => throw new InvalidOperationException($"Unknown source-range test case: {testCase.Name}"),
};
warmCase.Operations.ExtractSpan(
warmCase.Image, warmDirectory, null, [warmCase.PayloadEntry]);
Expand Down Expand Up @@ -187,6 +191,7 @@ private static IReadOnlyList<SpanCase> BuildCases(int payloadSize) => [
BuildPsdCase(payloadSize),
BuildPsbCase(payloadSize),
BuildJp2Case(payloadSize),
BuildJxlCase(payloadSize),
BuildMzCase(payloadSize),
BuildUImageCase(payloadSize),
BuildUefiFvCase(payloadSize),
Expand Down Expand Up @@ -441,6 +446,34 @@ private static SpanCase BuildJp2Case(int payloadSize) {
"JP2", new Jp2FormatDescriptor(), image, "codestream.j2c", payload);
}

private static SpanCase BuildJxlCase(int payloadSize) {
var payload = Pattern(payloadSize, 0xA4);
var firstLength = payload.Length / 2;
var secondLength = payload.Length - firstLength;

using var output = new MemoryStream();
output.Write([0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x4C, 0x20, 0x0D, 0x0A, 0x87, 0x0A]);

using (var firstBody = new MemoryStream()) {
Span<byte> index = stackalloc byte[4];
BinaryPrimitives.WriteUInt32BigEndian(index, 0);
firstBody.Write(index);
firstBody.Write(payload.AsSpan(0, firstLength));
WriteIsoBox(output, "jxlp", firstBody.GetBuffer().AsSpan(0, (int)firstBody.Length));
}

using (var secondBody = new MemoryStream()) {
Span<byte> index = stackalloc byte[4];
BinaryPrimitives.WriteUInt32BigEndian(index, 0x80000001);
secondBody.Write(index);
secondBody.Write(payload.AsSpan(firstLength, secondLength));
WriteIsoBox(output, "jxlp", secondBody.GetBuffer().AsSpan(0, (int)secondBody.Length));
}

return new SpanCase(
"JXL", new JxlFormatDescriptor(), output.ToArray(), "codestream.jxl", payload);
}

private static SpanCase BuildMzCase(int payloadSize) {
const int headerLength = 32;
var imageLength = headerLength + payloadSize;
Expand Down Expand Up @@ -551,6 +584,17 @@ private static byte[] Pattern(int length, byte seed) {
return result;
}

private static void WriteIsoBox(Stream output, string type, ReadOnlySpan<byte> body) {
if (type.Length != 4)
throw new ArgumentException("ISO box type must be exactly four characters.", nameof(type));

Span<byte> header = stackalloc byte[8];
BinaryPrimitives.WriteUInt32BigEndian(header[..4], checked((uint)(8 + body.Length)));
Encoding.ASCII.GetBytes(type, header[4..8]);
output.Write(header);
output.Write(body);
}

private static void WriteSupSegment(
Stream output, byte type, uint pts, ReadOnlySpan<byte> body, uint dts = 0) {
if (body.Length > ushort.MaxValue)
Expand Down
Loading