From 8e18810618e35119fae90b8ef476d4e29032ab71 Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Fri, 6 Nov 2026 17:00:17 +0530 Subject: [PATCH] Compare bounded decoding against libheif in CI CI now decodes every corpus file with decode_bounded at max side 6000 and 97 and compares the result with the strict libheif oracle, after applying Exif orientation, sRGB conversion and an area average. Generated fixtures cover odd-size crops, rotations, mirrors and nclx colour. decode_bounded now refuses clap crops, rotations and mirrors for which libheif converts 4:2:0 chroma to 4:4:4 first, so every image it decodes matches libheif. --- .github/workflows/tests.yml | 23 ++++ API.md | 9 +- TESTING.md | 54 +++++++++ scripts/incremental/generate-bounded.rb | 44 +++++++ scripts/incremental/primary-oracle.cc | 12 +- scripts/incremental/verify-bounded.rb | 152 ++++++++++++++++++++++++ src/bin/incremental-allocation.rs | 89 +++++++++----- src/bin/incremental-compare.rs | 91 ++++++++++++-- src/bounded.rs | 5 + src/bounded/grid.rs | 35 ++++++ src/bounded/tests.rs | 152 ++++++++++++++++++++++-- 11 files changed, 615 insertions(+), 51 deletions(-) create mode 100644 scripts/incremental/generate-bounded.rb create mode 100644 scripts/incremental/verify-bounded.rb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5031231..5b5169b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -151,6 +151,26 @@ jobs: REFERENCE_YUV="$prefix.yuv" target/release/incremental-check ".heic-test-assets/incremental-corpus/$name.heic" done + - name: Bounded decoder oracle comparisons + run: | + ruby scripts/incremental/generate-bounded.rb .heic-test-assets/incremental-corpus + required=() + for path in .heic-test-assets/bounded-corpus/pass-*.heic \ + .heic-test-assets/ente-test-fixtures/media/heic/v1/files/IMG_0983.HEIC \ + .heic-test-assets/ente-test-fixtures/media/heic/v1/files/1718_rotate_90_cw.HEIC; do + required+=(--require-pass "$path") + done + ruby scripts/incremental/verify-bounded.rb "${required[@]}" \ + .heic-test-runs/incremental-oracle/primary-oracle \ + target/release/incremental-allocation target/release/incremental-compare \ + .heic-test-assets/libheif/examples \ + .heic-test-assets/libheif/tests/data \ + .heic-test-assets/libheif/fuzzing/data/corpus \ + .heic-test-assets/ente-test-fixtures/media/heic/v1/files \ + .heic-test-assets/stress-corpus \ + .heic-test-assets/incremental-corpus \ + .heic-test-assets/bounded-corpus + - name: Upload verify report if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -159,5 +179,8 @@ jobs: path: | .heic-test-runs/verify/run/report.txt .heic-test-runs/incremental/*.json* + .heic-test-runs/bounded/*.json* + .heic-test-runs/bounded/summary.txt + .heic-test-assets/bounded-corpus/ .heic-test-assets/incremental-corpus/ if-no-files-found: ignore diff --git a/API.md b/API.md index 7a05f5c..dc1a991 100644 --- a/API.md +++ b/API.md @@ -432,7 +432,14 @@ The primary item may be a grid of uniform HEVC tiles or a single HEVC image (`hvc1`/`hev1`). Every coded item must be opaque 8-bit 4:2:0 HEVC with one IDR slice; WPP is supported. A single image is decoded as a grid with one tile. `clap`/`irot`/`imir` are applied, and Exif orientation applies when no -effective container orientation is present. RGB matrix/TRC ICC profiles and +effective container orientation is present. Container transforms for which +libheif first converts 4:2:0 chroma to 4:4:4 return `Unsupported`. Each is +judged at the size the image has when it applies: a `clap` with an odd left or +top edge, an anticlockwise 90° rotation (`irot` 1) of an odd width, a 180° +rotation with either side odd, an anticlockwise 270° rotation (`irot` 3) of an +odd height, or a mirror with either side odd. A display area that starts on an +odd source row or column is also refused, which is stricter than libheif when +a rotation or mirror comes before the crop. RGB matrix/TRC ICC profiles and SDR nclx colour with BT.709, BT.2020 or Display P3 primaries are converted to sRGB. HDR transfer functions, grayscale or lookup-table ICC profiles, unprofiled wide-gamut HEVC colour, alpha, and diff --git a/TESTING.md b/TESTING.md index bc8c542..54d562b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -162,6 +162,60 @@ libheif corpus and the ente fixtures. Generated reports and PNG artifacts are under `.heic-test-runs/`. Use `--keep-artifacts` with `verify` when debugging a pixel mismatch. +## Bounded decoder + +CI also compares `decode_bounded` with the strict libheif primary-image oracle +described in the next section. Build the oracle and generate the incremental +corpus as shown there, then run: + +```bash +cargo build --release --locked --features incremental-experiment --bins +ruby scripts/incremental/generate-bounded.rb .heic-test-assets/incremental-corpus +ruby scripts/incremental/verify-bounded.rb \ + --require-pass .heic-test-assets/bounded-corpus/pass-odd-width-crop.heic \ + .heic-test-runs/incremental-oracle/primary-oracle \ + target/release/incremental-allocation target/release/incremental-compare \ + .heic-test-assets/libheif/examples \ + .heic-test-assets/libheif/tests/data \ + .heic-test-assets/libheif/fuzzing/data/corpus \ + .heic-test-assets/ente-test-fixtures/media/heic/v1/files \ + .heic-test-assets/stress-corpus \ + .heic-test-assets/incremental-corpus \ + .heic-test-assets/bounded-corpus +``` + +`generate-bounded.rb` wraps the incremental corpus's 256×256 encode into the +ignored `.heic-test-assets/bounded-corpus`, or into an optional output folder. +These files are kept out of `incremental-corpus` because `verify.rb` requires +every file there to decode with the incremental experiment. The script refuses +an output folder that overlaps the incremental corpus, or that is not empty +and lacks the `.generate-bounded` marker it writes, and it only replaces the +`pass-*` and `refuse-*` files in its own folder. `refuse-*` files use rotations, mirrors or +grid canvases for which libheif converts to 4:4:4, so `decode_bounded` must +refuse them; if one decodes, the pixel comparison fails. `pass-*` files are +the matching controls and the Display P3 and BT.2020 nclx cases. CI passes +every `pass-*` file, `IMG_0983.HEIC` and `1718_rotate_90_cw.HEIC` with +`--require-pass`, which fails the run unless that file passes at both sizes. + +Every HEIC, HEIF and AVIF file under the given paths is decoded at max side +6000 and 97 with a 512 MiB budget (`BOUNDED_BUDGET_MIB` overrides it). The +runner denies any allocation that would take the live heap past the budget. +Each successful decode is compared with the oracle's image after the +comparator applies the Exif orientation reported by +`exif_orientation_hint_from_path`, converts an embedded ICC profile or the +item's BT.2020 or Display P3 nclx primaries to sRGB, and area-averages to the +output size. The `rgb8-rounding` gate described below applies. The Exif value +comes from the decoder's own parser and the colour conversion from moxcms; +the comparator's orientation mapping and area average are independent. + +Refusals with `Unsupported`, `LimitExceeded` or `MemoryBudgetExceeded` are +counted but do not fail the run. The run fails on a crash, a timeout, an +oracle timeout or crash, `AllocationFailed` for any file, any other error on a +file the oracle decodes, a successful decode of a file the strict oracle +rejects, a dimension or pixel mismatch, or a required file that does not pass. +Results and a summary are written under `.heic-test-runs/bounded`; +`BOUNDED_TEST_ROOT` overrides the folder. + ## Incremental bounded decoder The opt-in `incremental-experiment` feature is tested separately because diff --git a/scripts/incremental/generate-bounded.rb b/scripts/incremental/generate-bounded.rb new file mode 100644 index 0000000..f3e7212 --- /dev/null +++ b/scripts/incremental/generate-bounded.rb @@ -0,0 +1,44 @@ +require 'fileutils' +require 'open3' +require 'rbconfig' + +abort 'Usage: generate-bounded.rb INCREMENTAL_CORPUS_DIRECTORY [OUTPUT_DIRECTORY]' unless (1..2).cover?(ARGV.size) +corpus = File.expand_path(ARGV[0]) +source = File.join(corpus, 'direct.hevc') +abort "Missing #{source}; generate the incremental corpus first" unless File.file?(source) +root = File.expand_path('../..', __dir__) +output = File.expand_path(ARGV[1] || File.join(root, '.heic-test-assets/bounded-corpus')) +within = ->(path, directory) { path == directory || path.start_with?("#{directory}/") } +abort 'The output directory must not overlap the incremental corpus' if within.call(output, corpus) || within.call(corpus, output) +marker = File.join(output, '.generate-bounded') +if Dir.exist?(output) && !Dir.empty?(output) && !File.file?(marker) + abort "#{output} is not empty and was not created by generate-bounded.rb" +end +FileUtils.mkdir_p(output) +File.write(marker, '') +FileUtils.rm_f(Dir.glob(File.join(output, '{pass,refuse}-*.heic'))) + +cases = { + 'pass-odd-width-crop' => { 'CROP' => '131x192' }, + 'pass-odd-width-crop-rotate-270' => { 'CROP' => '131x192', 'ROTATION' => '3' }, + 'pass-odd-height-crop-rotate-90' => { 'CROP' => '192x129', 'ROTATION' => '1' }, + 'pass-odd-canvas-grid-rotate-270' => { 'GRID' => '1', 'CANVAS' => '255x256', 'ROTATION' => '3' }, + 'pass-nclx-display-p3' => { 'TILE_NCLX' => '12/13/6/0' }, + 'pass-nclx-bt2020' => { 'TILE_NCLX' => '9/1/9/0' }, + 'refuse-odd-width-crop-rotate-90' => { 'CROP' => '131x192', 'ROTATION' => '1' }, + 'refuse-odd-width-crop-rotate-180' => { 'CROP' => '131x192', 'ROTATION' => '2' }, + 'refuse-odd-height-crop-rotate-180' => { 'CROP' => '192x129', 'ROTATION' => '2' }, + 'refuse-odd-height-crop-rotate-270' => { 'CROP' => '192x129', 'ROTATION' => '3' }, + 'refuse-odd-width-crop-mirror-vertical-axis' => { 'CROP' => '131x192', 'MIRROR' => '0' }, + 'refuse-odd-width-crop-mirror-horizontal-axis' => { 'CROP' => '131x192', 'MIRROR' => '1' }, + 'refuse-odd-canvas-grid-rotate-180' => { 'GRID' => '1', 'CANVAS' => '255x256', 'ROTATION' => '2' } +} +keys = %w[GRID CANVAS CROP ROTATION MIRROR TILE_NCLX PRIMARY_NCLX REFERENCE_COUNT] +cases.each do |name, options| + path = File.join(output, "#{name}.heic") + environment = keys.to_h { |key| [key, nil] }.merge(options) + _, stderr, status = Open3.capture3(environment, RbConfig.ruby, File.join(__dir__, 'wrap.rb'), source, path, + '256', '256', '1', '1') + abort "wrap.rb failed for #{name}:\n#{stderr}" unless status.success? + puts "Generated #{path}" +end diff --git a/scripts/incremental/primary-oracle.cc b/scripts/incremental/primary-oracle.cc index 4b7ad93..cd97b58 100644 --- a/scripts/incremental/primary-oracle.cc +++ b/scripts/incremental/primary-oracle.cc @@ -45,9 +45,19 @@ int main(int argc, char **argv) { int warnings = heif_image_get_decoding_warnings(image.get(), 0, nullptr, 0); int width = heif_image_get_width(image.get(), heif_channel_interleaved); int height = heif_image_get_height(image.get(), heif_channel_interleaved); + std::string nclx_json = "null"; + heif_color_profile_nclx *nclx = nullptr; + if (heif_image_handle_get_nclx_color_profile(handle.get(), &nclx).code == heif_error_Ok && nclx) { + nclx_json = "{\"primaries\":" + std::to_string(static_cast(nclx->color_primaries)) + + ",\"transfer\":" + std::to_string(static_cast(nclx->transfer_characteristics)) + + ",\"matrix\":" + std::to_string(static_cast(nclx->matrix_coefficients)) + + ",\"full_range\":" + (nclx->full_range_flag ? "true" : "false") + "}"; + heif_nclx_color_profile_free(nclx); + } std::cout << "{\"primary_id\":" << primary << ",\"width\":" << width << ",\"height\":" << height << ",\"warnings\":" << warnings - << ",\"strict\":" << (options->strict_decoding ? "true" : "false") << "}\n"; + << ",\"strict\":" << (options->strict_decoding ? "true" : "false") + << ",\"nclx\":" << nclx_json << "}\n"; if (argc >= 3 && std::string(argv[2]) != "-") { std::vector icc(heif_image_get_raw_color_profile_size(image.get())); if (!icc.empty()) { diff --git a/scripts/incremental/verify-bounded.rb b/scripts/incremental/verify-bounded.rb new file mode 100644 index 0000000..a0f0668 --- /dev/null +++ b/scripts/incremental/verify-bounded.rb @@ -0,0 +1,152 @@ +require 'digest' +require 'fileutils' +require 'json' +require 'open3' +require 'timeout' + +required = [] +arguments = [] +until ARGV.empty? + argument = ARGV.shift + if argument == '--require-pass' + abort '--require-pass needs a file' if ARGV.empty? + required << File.expand_path(ARGV.shift) + else + arguments << argument + end +end +abort 'Usage: verify-bounded.rb [--require-pass FILE]... ORACLE RUNNER COMPARATOR INPUT_OR_DIRECTORY...' if arguments.size < 4 +oracle, runner, comparator, *sources = arguments.map { |p| File.expand_path(p) } +root = File.expand_path('../..', __dir__) +output = File.expand_path(ENV.fetch('BOUNDED_TEST_ROOT', File.join(root, '.heic-test-runs/bounded'))) +budget_mib = Integer(ENV.fetch('BOUNDED_BUDGET_MIB', '512')) +sides = [6000, 97] +refusals = %w[unsupported limit_exceeded memory_budget_exceeded] +fatal = %w[timeout crash allocation_failed unexpected_error bounded_only dimension_mismatch pixel_mismatch + oracle_timeout oracle_crash] +FileUtils.mkdir_p(output) + +inputs = sources.flat_map do |source| + if File.directory?(source) + Dir.glob(File.join(source, '**', '*')).select { |p| File.file?(p) && p.match?(/\.(heic|heif|avif)\z/i) } + else + [source] + end +end.uniq.sort +abort 'No inputs found' if inputs.empty? +missing = required - inputs +abort "Required inputs not found: #{missing.join(', ')}" unless missing.empty? + +def run(*args) + Open3.popen3(*args, pgroup: true) do |input, out, err, thread| + input.close + stdout = Thread.new { out.read } + stderr = Thread.new { err.read } + begin + status = Timeout.timeout(120) { thread.value } + [status, stdout.value, stderr.value] + rescue Timeout::Error + Process.kill('KILL', -thread.pid) + thread.value + [nil, stdout.value, stderr.value] + end + end +end + +def oriented(width, height, orientation) + orientation >= 5 ? [height, width] : [width, height] +end + +manifest = { + binaries: [oracle, runner, comparator].to_h { |p| [p, Digest::SHA256.file(p).hexdigest] }, + inputs: inputs.size, + required: required, + budget_mib: budget_mib, + sides: sides, + profile: 'rgb8-rounding' +} +File.write(File.join(output, 'manifest.json'), JSON.pretty_generate(manifest)) +counts = Hash.new(0) +failures = [] +File.open(File.join(output, 'results.jsonl'), 'w') do |report| + inputs.each do |path| + dir = File.join(output, Digest::SHA256.hexdigest(path)) + FileUtils.mkdir_p(dir) + png = File.join(dir, 'reference.png') + status, stdout, stderr = run(oracle, path, png, 'strict') + oracle_outcome = + if status.nil? + 'oracle_timeout' + elsif status.exitstatus.nil? || ![0, 1, 2].include?(status.exitstatus) + 'oracle_crash' + end + oracle_error = stderr.strip unless status&.success? + reference = status&.success? ? JSON.parse(stdout) : nil + if oracle_outcome + counts[oracle_outcome] += 1 + failures << "#{path}: #{oracle_outcome}: #{oracle_error}" + end + sides.each do |side| + raw = File.join(dir, 'actual.rgb') + FileUtils.rm_f(raw) + status, stdout, stderr = run(runner, 'decode-bounded', path, side.to_s, budget_mib.to_s, raw) + decoded = begin + JSON.parse(stdout.lines.last.to_s) + rescue JSON::ParserError + nil + end + entry = { path: path, side: side, reference: reference, oracle_error: oracle_error, decoded: decoded } + outcome = + if status.nil? + 'timeout' + elsif decoded.nil? || ![0, 1].include?(status.exitstatus) + entry[:stderr] = stderr.lines.last(20).join + 'crash' + elsif status.exitstatus == 1 + if decoded['kind'] == 'allocation_failed' + 'allocation_failed' + elsif reference.nil? + 'both_refused' + elsif refusals.include?(decoded['kind']) + 'refused' + else + 'unexpected_error' + end + elsif reference.nil? + 'bounded_only' + else + orientation = decoded.fetch('exif_orientation') + expected = oriented(reference.fetch('width'), reference.fetch('height'), orientation) + if [decoded.fetch('original_width'), decoded.fetch('original_height')] != expected + entry[:expected_original] = expected + 'dimension_mismatch' + else + primaries = reference.dig('nclx', 'primaries') || 0 + status, stdout, stderr = run(comparator, 'png', png, raw, decoded.fetch('width').to_s, + decoded.fetch('height').to_s, side.to_s, 'rgb8-rounding', + orientation.to_s, primaries.to_s) + entry[:metrics] = JSON.parse(stdout) rescue nil + entry[:comparator] = stderr.strip + status&.success? ? 'pass' : 'pixel_mismatch' + end + end + entry[:outcome] = outcome + report.puts(JSON.generate(entry)) + report.flush + FileUtils.rm_f(raw) + counts[outcome] += 1 + if fatal.include?(outcome) + detail = decoded&.fetch('error', nil) || entry[:comparator] || entry[:stderr] || oracle_error + failures << "#{path} (side #{side}): #{outcome}: #{detail}" + elsif required.include?(path) && outcome != 'pass' + failures << "#{path} (side #{side}): required to pass but #{outcome}: #{decoded&.fetch('error', nil)}" + end + end + FileUtils.rm_f(png) + end +end +summary = counts.sort.map { |outcome, count| "#{outcome}=#{count}" }.join(' ') +File.write(File.join(output, 'summary.txt'), "#{summary}\n#{failures.join("\n")}\n") +puts summary +failures.each { |failure| puts "FAIL #{failure}" } +exit(failures.empty? && counts['pass'].positive? ? 0 : 1) diff --git a/src/bin/incremental-allocation.rs b/src/bin/incremental-allocation.rs index 5607957..29b5601 100644 --- a/src/bin/incremental-allocation.rs +++ b/src/bin/incremental-allocation.rs @@ -1,16 +1,34 @@ #[path = "support/allocation.rs"] mod allocation; use allocation::*; +use heic_decoder::BoundedDecodeError; use std::path::Path; use std::sync::atomic::Ordering; use std::time::Instant; +fn kind(error: &BoundedDecodeError) -> &'static str { + match error { + BoundedDecodeError::InvalidOptions(_) => "invalid_options", + BoundedDecodeError::Malformed(_) => "malformed", + BoundedDecodeError::Unsupported(_) => "unsupported", + BoundedDecodeError::LimitExceeded(_) => "limit_exceeded", + BoundedDecodeError::MemoryBudgetExceeded { .. } => "memory_budget_exceeded", + BoundedDecodeError::AllocationFailed => "allocation_failed", + BoundedDecodeError::Io(_) => "io", + BoundedDecodeError::Decode(_) => "decode", + } +} + fn main() { let args: Vec = std::env::args().collect(); - if !(3..=6).contains(&args.len()) || !matches!(args[1].as_str(), "bounded" | "bytes" | "normal") + if !(3..=6).contains(&args.len()) + || !matches!( + args[1].as_str(), + "bounded" | "bytes" | "normal" | "decode-bounded" + ) { eprintln!( - "Usage: incremental-allocation bounded|bytes|normal INPUT [MAX_SIDE] [MIB] [RGB_OUTPUT]" + "Usage: incremental-allocation bounded|bytes|normal|decode-bounded INPUT [MAX_SIDE] [MIB] [RGB_OUTPUT]" ); std::process::exit(2); } @@ -29,6 +47,14 @@ fn main() { } else { None }; + let exif_orientation = heic_decoder::exif_orientation_hint_from_path(path) + .ok() + .and_then(|hint| hint.orientation_to_apply()) + .unwrap_or(1); + let options = heic_decoder::BoundedDecodeOptions { + max_side: side, + max_memory_bytes: mib * 1024 * 1024, + }; let baseline = LIVE.load(Ordering::Relaxed); if !mode.starts_with("normal") { MAX_LIVE.store(baseline + mib * 1024 * 1024, Ordering::Relaxed); @@ -36,29 +62,34 @@ fn main() { PEAK.store(baseline, Ordering::Relaxed); ACTIVE.store(true, Ordering::Relaxed); let start = Instant::now(); - let result = if mode == "normal" { - heic_decoder::decode_path_to_rgb8(path).map_err(|e| e.to_string()) - } else { - let input = borrowed - .as_ref() - .map_or(heic_decoder::BoundedInput::Path(path), |bytes| { - heic_decoder::BoundedInput::Bytes(bytes) - }); - heic_decoder::decode_incremental_experiment( - input, - heic_decoder::BoundedDecodeOptions { - max_side: side, - max_memory_bytes: mib * 1024 * 1024, - }, - ) - .map(|(image, stats)| { - eprintln!( - "decoded_rows={} sao_edge_components={} sao_band_components={}", - stats[0], stats[1], stats[2] - ); - image.image - }) - .map_err(|e| e.to_string()) + let result = match mode { + "normal" => heic_decoder::decode_path_to_rgb8(path) + .map(|image| { + let dimensions = (image.width, image.height); + (image, dimensions) + }) + .map_err(|e| (e.to_string(), "decode")), + "decode-bounded" => { + heic_decoder::decode_bounded(heic_decoder::BoundedInput::Path(path), options) + .map(|bounded| (bounded.image, bounded.original_dimensions)) + .map_err(|e| (e.to_string(), kind(&e))) + } + _ => { + let input = borrowed + .as_ref() + .map_or(heic_decoder::BoundedInput::Path(path), |bytes| { + heic_decoder::BoundedInput::Bytes(bytes) + }); + heic_decoder::decode_incremental_experiment(input, options) + .map(|(image, stats)| { + eprintln!( + "decoded_rows={} sao_edge_components={} sao_band_components={}", + stats[0], stats[1], stats[2] + ); + (image.image, image.original_dimensions) + }) + .map_err(|e| (e.to_string(), kind(&e))) + } }; let elapsed = start.elapsed(); ACTIVE.store(false, Ordering::Relaxed); @@ -71,9 +102,9 @@ fn main() { let largest = LARGEST.load(Ordering::Relaxed); let workers = THREADS.load(Ordering::Relaxed).saturating_sub(1); match result { - Ok(image) => { + Ok((image, (original_width, original_height))) => { println!( - "{{\"mode\":\"{mode}\",\"side\":{side},\"budget_mib\":{mib},\"width\":{},\"height\":{},\"milliseconds\":{:.3},\"peak_bytes\":{peak},\"largest_bytes\":{largest},\"allocating_workers\":{workers}}}", + "{{\"mode\":\"{mode}\",\"side\":{side},\"budget_mib\":{mib},\"width\":{},\"height\":{},\"original_width\":{original_width},\"original_height\":{original_height},\"exif_orientation\":{exif_orientation},\"milliseconds\":{:.3},\"peak_bytes\":{peak},\"largest_bytes\":{largest},\"allocating_workers\":{workers}}}", image.width, image.height, elapsed.as_secs_f64() * 1000.0 @@ -85,9 +116,9 @@ fn main() { std::fs::write(output, image.pixels).unwrap(); } } - Err(error) => { + Err((error, kind)) => { println!( - "{{\"error\":\"{error}\",\"milliseconds\":{:.3},\"peak_bytes\":{peak},\"largest_bytes\":{largest}}}", + "{{\"error\":{error:?},\"kind\":\"{kind}\",\"milliseconds\":{:.3},\"peak_bytes\":{peak},\"largest_bytes\":{largest}}}", elapsed.as_secs_f64() * 1000.0 ); std::process::exit(1); diff --git a/src/bin/incremental-compare.rs b/src/bin/incremental-compare.rs index e00468e..f0247fd 100644 --- a/src/bin/incremental-compare.rs +++ b/src/bin/incremental-compare.rs @@ -9,7 +9,7 @@ struct Reference { transformed: bool, } -fn read_reference(path: &str) -> Result> { +fn read_reference(path: &str, nclx_primaries: u8) -> Result> { let mut decoder = png::Decoder::new(BufReader::new(File::open(path)?)); decoder.set_transformations(png::Transformations::EXPAND); let mut reader = decoder.read_info()?; @@ -33,9 +33,20 @@ fn read_reference(path: &str) -> Result> { } _ => return Err("Expected RGB reference".into()), }; - let transformed = icc.is_some(); - let pixels = if let Some(icc) = icc { - let profile = moxcms::ColorProfile::new_from_slice(&icc)?; + let profile = match icc { + Some(icc) => Some(moxcms::ColorProfile::new_from_slice(&icc)?), + None if matches!(nclx_primaries, 9 | 12) => { + Some(moxcms::ColorProfile::new_from_cicp(moxcms::CicpProfile { + color_primaries: moxcms::CicpColorPrimaries::try_from(nclx_primaries)?, + transfer_characteristics: moxcms::TransferCharacteristics::Srgb, + matrix_coefficients: moxcms::MatrixCoefficients::Identity, + full_range: true, + })) + } + None => None, + }; + let transformed = profile.is_some(); + let pixels = if let Some(profile) = profile { let transform = profile.create_transform_8bit( moxcms::Layout::Rgb, &moxcms::ColorProfile::new_srgb(), @@ -56,6 +67,33 @@ fn read_reference(path: &str) -> Result> { }) } +fn orient(source: &[u8], width: usize, height: usize, orientation: u8) -> (Vec, usize, usize) { + let (ow, oh) = if orientation >= 5 { + (height, width) + } else { + (width, height) + }; + let mut output = vec![0; source.len()]; + for y in 0..oh { + for x in 0..ow { + let (sx, sy) = match orientation { + 2 => (width - 1 - x, y), + 3 => (width - 1 - x, height - 1 - y), + 4 => (x, height - 1 - y), + 5 => (y, x), + 6 => (y, height - 1 - x), + 7 => (width - 1 - y, height - 1 - x), + 8 => (width - 1 - y, x), + _ => (x, y), + }; + let from = (sy * width + sx) * 3; + let to = (y * ow + x) * 3; + output[to..to + 3].copy_from_slice(&source[from..from + 3]); + } + } + (output, ow, oh) +} + fn area_reference(source: &[u8], sw: usize, sh: usize, w: usize, h: usize) -> Vec { if (sw, sh) == (w, h) { return source.to_vec(); @@ -200,7 +238,7 @@ impl Profile { fn main() -> Result<(), Box> { let args: Vec = std::env::args().collect(); if args.len() < 7 { - return Err("Usage: comparator raw EXPECTED ACTUAL WIDTH CHANNELS PROFILE | png REFERENCE ACTUAL WIDTH HEIGHT SIDE PROFILE".into()); + return Err("Usage: comparator raw EXPECTED ACTUAL WIDTH CHANNELS PROFILE | png REFERENCE ACTUAL WIDTH HEIGHT SIDE PROFILE [ORIENTATION NCLX_PRIMARIES]".into()); } if args[1] == "raw" { let expected = std::fs::read(&args[2])?; @@ -220,14 +258,23 @@ fn main() -> Result<(), Box> { if !profile.accepts(&expected, &actual, channels) { return Err("Pixel comparison failed".into()); } - } else if args[1] == "png" && args.len() == 8 { + } else if args[1] == "png" && matches!(args.len(), 8 | 10) { let profile = Profile::parse(&args[7])?; + let (orientation, nclx_primaries) = if args.len() == 10 { + (args[8].parse::()?, args[9].parse::()?) + } else { + (1, 0) + }; + if !(1..=8).contains(&orientation) { + return Err("Orientation must be 1 through 8".into()); + } let Reference { - width: sw, - height: sh, - pixels: source, + width: reference_width, + height: reference_height, + pixels, transformed, - } = read_reference(&args[2])?; + } = read_reference(&args[2], nclx_primaries)?; + let (source, sw, sh) = orient(&pixels, reference_width, reference_height, orientation); let actual = std::fs::read(&args[3])?; let width: usize = args[4].parse()?; let height: usize = args[5].parse()?; @@ -246,7 +293,9 @@ fn main() -> Result<(), Box> { .into()); } let expected = area_reference(&source, sw, sh, width, height); - eprintln!("reference={sw}x{sh} output={width}x{height} icc_transformed={transformed}"); + eprintln!( + "reference={sw}x{sh} output={width}x{height} orientation={orientation} color_transformed={transformed}" + ); println!("{}", metrics(&expected, &actual, width, 3)); if !profile.accepts(&expected, &actual, 3) { return Err("Pixel comparison failed".into()); @@ -274,6 +323,26 @@ mod tests { assert!(metrics(&[0, 0, 0, 255], &[0, 0, 0, 254], 1, 4).contains("\"alpha_errors\":1,")); } + #[test] + fn exif_orientations_map_to_display_layout() { + let source: Vec = (0..6u8).flat_map(|v| [v; 3]).collect(); + for (orientation, dimensions, expected) in [ + (1, (3, 2), [0, 1, 2, 3, 4, 5]), + (2, (3, 2), [2, 1, 0, 5, 4, 3]), + (3, (3, 2), [5, 4, 3, 2, 1, 0]), + (4, (3, 2), [3, 4, 5, 0, 1, 2]), + (5, (2, 3), [0, 3, 1, 4, 2, 5]), + (6, (2, 3), [3, 0, 4, 1, 5, 2]), + (7, (2, 3), [5, 2, 4, 1, 3, 0]), + (8, (2, 3), [2, 5, 1, 4, 0, 3]), + ] { + let (pixels, width, height) = orient(&source, 3, 2, orientation); + assert_eq!((width, height), dimensions); + let values: Vec = pixels.chunks_exact(3).map(|p| p[0]).collect(); + assert_eq!(values, expected, "orientation {orientation}"); + } + } + #[test] fn independent_area_average() { assert_eq!( diff --git a/src/bounded.rs b/src/bounded.rs index b1b448a..feb73aa 100644 --- a/src/bounded.rs +++ b/src/bounded.rs @@ -131,6 +131,11 @@ pub fn decode_bounded( let index = container::Index::parse(&metadata, source.len(), &budget)?; let grid = grid::Grid::preflight(&index, &mut source, &budget)?; let layout = resample::Layout::new(&grid, options.max_side)?; + if layout.left % 2 != 0 || layout.top % 2 != 0 || !grid.transforms_keep_subsampled_chroma()? { + return Err(BoundedDecodeError::Unsupported( + "transform that needs 4:4:4 chroma", + )); + } let first = &grid.tiles[0]; let mut color = first.color.clone(); if grid diff --git a/src/bounded/grid.rs b/src/bounded/grid.rs index 5dfdd9f..3183592 100644 --- a/src/bounded/grid.rs +++ b/src/bounded/grid.rs @@ -225,6 +225,41 @@ impl<'a> Tile<'a> { } impl<'a> Grid<'a> { + pub(super) fn transforms_keep_subsampled_chroma(&self) -> Result { + let (mut width, mut height) = (self.width, self.height); + for transform in self.properties.transforms.iter().flatten() { + let (odd_width, odd_height) = (width % 2 != 0, height % 2 != 0); + match transform { + Transform::CleanAperture(aperture) => { + let crop = crate::clean_aperture_crop_bounds(width, height, *aperture) + .map_err(|_| Error::Malformed("display crop"))?; + if crop.left % 2 != 0 || crop.top % 2 != 0 { + return Ok(false); + } + (width, height) = (crop.width, crop.height); + } + Transform::Rotation(rotation) => { + let degrees = rotation.rotation_ccw_degrees % 360; + if (degrees == 90 && odd_width) + || (degrees == 180 && (odd_width || odd_height)) + || (degrees == 270 && odd_height) + { + return Ok(false); + } + if degrees % 180 == 90 { + (width, height) = (height, width); + } + } + Transform::Mirror(_) => { + if odd_width || odd_height { + return Ok(false); + } + } + } + } + Ok(true) + } + pub(super) fn preflight( index: &Index<'a>, source: &mut Source<'_>, diff --git a/src/bounded/tests.rs b/src/bounded/tests.rs index a468989..4ac6c8c 100644 --- a/src/bounded/tests.rs +++ b/src/bounded/tests.rs @@ -76,15 +76,37 @@ fn fixture_with_exif_info( tile_extras: &[usize], orientation: Option, exif_info: &[u8], +) -> Vec { + fixture_with_canvas( + configs, + payload, + extra_properties, + primary_extras, + tile_extras, + orientation, + exif_info, + (configs.len() as u32 * 16 - 1, 15), + ) +} + +#[allow(clippy::too_many_arguments)] +fn fixture_with_canvas( + configs: &[Vec], + payload: &[u8], + extra_properties: &[Vec], + primary_extras: &[usize], + tile_extras: &[usize], + orientation: Option, + exif_info: &[u8], + (width, height): (u32, u32), ) -> Vec { let count = configs.len() as u16; let primary = count + 1; let item_count = primary + u16::from(orientation.is_some()); - let width = u32::from(count) * 16 - 1; let descriptor = [ vec![0, 0, 0, count as u8 - 1], (width as u16).to_be_bytes().to_vec(), - 15u16.to_be_bytes().to_vec(), + (height as u16).to_be_bytes().to_vec(), ] .concat(); let payload = [&(payload.len() as u32).to_be_bytes()[..], payload].concat(); @@ -157,7 +179,7 @@ fn fixture_with_exif_info( properties.extend(full_box( b"ispe", 0, - &[width.to_be_bytes(), 15u32.to_be_bytes()].concat(), + &[width.to_be_bytes(), height.to_be_bytes()].concat(), )); for config in configs { properties.extend(box_bytes(b"hvcC", config)); @@ -1118,13 +1140,27 @@ fn mirrored_asymmetric_pixels_match_normal_decode() { let config = configuration(&nal_sets()[1]); for rotation in 0..4 { for mirror in 0..2 { - let bytes = fixture( - &[config.clone(), config.clone()], + let configs = [config.clone(), config.clone()]; + let transforms = [ + box_bytes(b"irot", &[rotation]), + box_bytes(b"imir", &[mirror]), + ]; + let odd = fixture(&configs, &asymmetric_payload(), &transforms); + assert!(matches!( + decode_bounded(BoundedInput::Bytes(&odd), Default::default()), + Err(BoundedDecodeError::Unsupported( + "transform that needs 4:4:4 chroma" + )) + )); + let bytes = fixture_with_canvas( + &configs, &asymmetric_payload(), - &[ - box_bytes(b"irot", &[rotation]), - box_bytes(b"imir", &[mirror]), - ], + &transforms, + &[0, 1], + &[], + None, + b"Exif\0", + (30, 14), ); let normal = crate::decode_bytes_to_rgb8(&bytes).unwrap(); assert!( @@ -1621,3 +1657,101 @@ fn single_image_payloads_are_bounded_by_the_budget_instead_of_the_tile_cap() { }) )); } + +#[cfg(not(feature = "decoder-tracing"))] +#[test] +fn transforms_that_need_full_chroma_are_refused() { + let config = configuration(&nal_sets()[1]); + let payload = asymmetric_payload(); + let clap = |width: i32, height: i32, horizontal_offset: [i32; 2], vertical_offset: [i32; 2]| { + let crop: Vec<_> = [ + width, + 1, + height, + 1, + horizontal_offset[0], + horizontal_offset[1], + vertical_offset[0], + vertical_offset[1], + ] + .into_iter() + .flat_map(i32::to_be_bytes) + .collect(); + box_bytes(b"clap", &crop) + }; + let centred = [0, 1]; + let back_half = [-1, 2]; + let mirror = |axis| box_bytes(b"imir", &[axis]); + let quarter_turn = box_bytes(b"irot", &[1]); + let half_turn = box_bytes(b"irot", &[2]); + let three_quarter_turn = box_bytes(b"irot", &[3]); + let odd_width = clap(11, 12, back_half, centred); + let odd_height = clap(12, 11, centred, back_half); + for (properties, refused) in [ + (vec![clap(12, 12, centred, centred)], false), + ( + vec![clap(12, 12, centred, centred), quarter_turn.clone()], + false, + ), + ( + vec![half_turn.clone(), clap(12, 12, centred, centred)], + false, + ), + (vec![odd_width.clone()], false), + (vec![odd_width.clone(), three_quarter_turn.clone()], false), + (vec![odd_height.clone()], false), + (vec![odd_height.clone(), quarter_turn.clone()], false), + (vec![clap(12, 10, centred, centred)], true), + (vec![clap(10, 12, centred, centred)], true), + ( + vec![quarter_turn.clone(), clap(12, 10, centred, centred)], + true, + ), + (vec![half_turn.clone(), clap(11, 12, [1, 2], centred)], true), + (vec![half_turn.clone(), odd_width.clone()], true), + (vec![odd_width.clone(), quarter_turn.clone()], true), + (vec![odd_width.clone(), half_turn.clone()], true), + (vec![odd_width.clone(), mirror(0)], true), + (vec![odd_width, mirror(1)], true), + (vec![odd_height.clone(), half_turn], true), + (vec![odd_height.clone(), three_quarter_turn], true), + (vec![odd_height.clone(), mirror(0)], true), + (vec![odd_height, mirror(1)], true), + ] { + for primary in [Primary::Single, Primary::Grid] { + let bytes = coded_fixture(primary, &config, (16, 16), &[&payload], &properties, None); + let result = decode_bounded(BoundedInput::Bytes(&bytes), Default::default()); + if refused { + assert!( + matches!( + result, + Err(BoundedDecodeError::Unsupported( + "transform that needs 4:4:4 chroma" + )) + ), + "{result:?}" + ); + } else { + assert!(result.is_ok(), "{result:?}"); + } + } + } + let rotated_then_cropped = fixture_with_canvas( + &[config.clone(), config], + &payload, + &[quarter_turn, clap(10, 12, centred, centred)], + &[0, 1], + &[], + None, + b"Exif\0", + (28, 14), + ); + let bounded = decode_bounded( + BoundedInput::Bytes(&rotated_then_cropped), + Default::default(), + ) + .unwrap(); + let normal = crate::decode_bytes_to_rgb8(&rotated_then_cropped).unwrap(); + assert_eq!((bounded.image.width, bounded.image.height), (10, 12)); + assert_eq!(bounded.image.pixels, normal.pixels); +}