diff --git a/.gitattributes b/.gitattributes index 0fc74d72b..d8aed3ff5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -16,6 +16,9 @@ Jenkinsfile text *.sh text eol=lf # Denote all files that are truly binary and should not be modified. -*.png binary -*.hdr binary +*.astc binary +*.astcm binary *.exe binary +*.hdr binary +*.png binary +*.zip binary diff --git a/.github/workflows/build_test_rdo.yaml b/.github/workflows/build_test_rdo.yaml new file mode 100644 index 000000000..4d1a9ccb3 --- /dev/null +++ b/.github/workflows/build_test_rdo.yaml @@ -0,0 +1,67 @@ +name: Build and test RDO +run-name: Build and test RDO PR${{ github.event.number }} + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + +jobs: + build-ubuntu-x64-clang-rdo: + name: Ubuntu x64 Clang RDO (${{ matrix.config }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - config: Release + build_type: Release + extra_flags: '' + - config: ASAN + build_type: RelWithDebInfo + extra_flags: -DASTCENC_ASAN=ON + - config: UBSAN + build_type: RelWithDebInfo + extra_flags: -DASTCENC_UBSAN=ON + env: + CXX: clang++ + steps: + - name: Git checkout + uses: actions/checkout@v6 + with: + submodules: 'true' + + - name: Install LLD + run: | + sudo apt-get update + sudo apt-get install -y lld + + - name: Build and test RDO + shell: bash + run: | + cmake -S . -B build -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DASTCENC_ISA_AVX2=ON \ + -DASTCENC_RDO=ON -DASTCENC_UNITTEST=ON ${{ matrix.extra_flags }} + cmake --build build --target astcenc-avx2 test-unit-avx2 -j4 + ctest --test-dir build --output-on-failure --no-tests=error + + BINARY=build/Source/astcenc-avx2 + OUTPUT_DIR=build/Images + mkdir -p "$OUTPUT_DIR" + for IMAGE in Test/Images/Small/LDR-*/*.png Test/Images/Small/LDRS-*/*.png; do + PROFILE=l + case "$IMAGE" in + */LDRS-*/*) PROFILE=s ;; + esac + + IMAGE_NAME="${IMAGE##*/}" + for BLOCK in 4x4 5x5 6x6; do + OUTPUT="$OUTPUT_DIR/$IMAGE_NAME-$BLOCK" + "$BINARY" "-c$PROFILE" "$IMAGE" "$OUTPUT.astcm" \ + "$BLOCK" -thorough -rdo-lambda 0.2 -j 2 + "$BINARY" "-d$PROFILE" "$OUTPUT.astcm" "$OUTPUT.png" -j 2 + done + done diff --git a/CMakeLists.txt b/CMakeLists.txt index 6721adae5..465a9f2b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,11 +55,22 @@ option(ASTCENC_UBSAN "Enable astcenc builds with undefined behavior sanitizer") option(ASTCENC_UNITTEST "Enable astcenc builds with unit tests") option(ASTCENC_INVARIANCE "Enable astcenc floating point invariance" ON) option(ASTCENC_CLI "Enable build of astcenc command line tools" ON) +option(ASTCENC_RDO "Enable RDO (Experimental)" OFF) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + option(ASTCENC_RDO_SIZE "Enable RDO size reporting using zlib" OFF) +endif() +if(NOT ASTCENC_RDO) + set(ASTCENC_RDO_SIZE OFF CACHE BOOL "Enable RDO size reporting using zlib" FORCE) +endif() option(ASTCENC_X86_GATHERS "Enable use of native x86 gathers" ON) option(ASTCENC_WERROR "Force builds to treat warnings as errors" ON) set(ASTCENC_CLI_POSTFIX "" CACHE STRING "Custom postfix for command line tool binary names") +if(ASTCENC_RDO_SIZE) + find_package(ZLIB REQUIRED) +endif() + if(ASTCENC_ASAN AND ASTCENC_TSAN) message(FATAL_ERROR "ASTCENC_ASAN and ASTCENC_TSAN cannot be enabled together.") endif() @@ -166,6 +177,8 @@ if(${ASTCENC_BIG_ENDIAN}) endif() printopt("Shared libs " ${ASTCENC_SHAREDLIB}) printopt("Decompressor " ${ASTCENC_DECOMPRESSOR}) +printopt("RDO (Experimental) " ${ASTCENC_RDO}) +printopt("RDO size reporting " ${ASTCENC_RDO_SIZE}) message(STATUS "Developer options") printopt("Diagnostics " ${ASTCENC_DIAGNOSTICS}) printopt("ASAN " ${ASTCENC_ASAN}) diff --git a/Docs/FileFormat.md b/Docs/FileFormat.md index 975dd0b71..c0f1b26f8 100644 --- a/Docs/FileFormat.md +++ b/Docs/FileFormat.md @@ -1,4 +1,4 @@ -# The .astc File Format +# The .astc and .astcm File Formats The default file format for compressed textures generated by `astcenc`, as well as from many other ASTC compressors, is the `.astc` format. This is a very @@ -66,6 +66,41 @@ The binary payload is a byte stream that immediately follows the header. It contains 16 bytes per compressed block. The number of compressed blocks is determined from the header information. +The .astcm format +================= + +The `.astcm` format stores the same encoded ASTC blocks in an order intended +to improve locality for subsequent lossless compression. It supports 2D +images and 2D block footprints, including non-square footprints. It changes +neither the encoded blocks nor the decoded image quality. + +The header has the same 16-byte layout as `.astc`, with the four magic bytes +replaced by the ASCII bytes `ASCM` (`0x41, 0x53, 0x43, 0x4D`). Both `block_z` +and `dim_z` must be one. Image dimensions must be nonzero and fit in 24 bits. +As with `.astc`, the color profile is supplied separately by the application. + +The block grid is divided into bricks of 8 by 8 ASTC blocks. Bricks are stored +in raster order, with X changing fastest. Blocks inside each brick are stored +in Morton order, using the even bits of the Morton index for the local X +coordinate and the odd bits for the local Y coordinate: + +``` +for brick_y in range(0, blocks_y, 8): + for brick_x in range(0, blocks_x, 8): + for morton in range(64): + x = brick_x + sum(((morton >> (2 * i)) & 1) << i for i in range(3)) + y = brick_y + sum(((morton >> (2 * i + 1)) & 1) << i for i in range(3)) + if x < blocks_x and y < blocks_y: + write_block(x, y) +``` + +Blocks outside the image's block grid are skipped, so edge bricks add no +padding. The payload size is exactly `ceil(dim_x / block_x) * ceil(dim_y / +block_y) * 16` bytes. Each block retains its original byte order. + +The codec API continues to use raster-ordered blocks. The CLI loader restores +raster order, and the CLI writer applies the Morton permutation when saving. + - - - _Copyright © 2020-2026, Arm Limited and contributors._ diff --git a/Docs/RDO.md b/Docs/RDO.md new file mode 100644 index 000000000..1145e19df --- /dev/null +++ b/Docs/RDO.md @@ -0,0 +1,160 @@ +# Rate-distortion optimization + +Rate-distortion optimization (RDO) trades image quality for smaller files after +applying additional lossless compression to compressed ASTC files. It is used +to reduce the size of application distribution bundles, providing finer-grained +control over storage size than simply reducing ASTC block size. + +> [!NOTE] +> The RDO implementation in `astcenc` currently supports 2D LDR textures, using +> either the linear or sRGB color profile. + +## How RDO works + +ASTC compression is a fixed-rate compression that encodes every block into 16 +bytes of data. For a given image size and block footprint, the ASTC payload +data has a fixed size. RDO is a post-process that changes the compressed ASTC +bitstream to make byte patterns more similar to nearby bytes in the stream. +This makes the data more compressible when applying subsequent lossless +compression, such as ZIP or LZ4. + +The RDO transform is lossy, sacrificing image quality to improve data +compressibility. The amount of allowed quality loss is controlled by a lambda +setting, where a higher lambda values allow more image distortion in exchange +for a more compressible pattern. Lambda values between 0.0 (no RDO) and 0.3 +(moderate RDO) give a reasoanble tradeoff between quality and storage. Lambda +values above 0.3 quickly loose image quality for relatively little saving in +packaged file size, and it is usally a better option to choose a larger ASTC +block footprint. + +The ASTC blocks after RDO has been applied are still standard ASTC data blocks. +RDO does not require a special GPU decoder, and it does not change the number of +blocks or the size of each block. To realize the storage saving, additional +lossless compression must be applied in your asset packaging pipeline. For +Android, this is compression is commonly applied when generating an Application +APK (application bundle) or AAB (asset bundle). + +### Using RDO versus changing the ASTC block size + +When using ASTC, you have two controls that can reduce packaged file size. You +can choose to use a larger ASTC block size, which simply reduces the amount of +data that needs to be packaged, or you can choose to use RDO to make data more +compressible. + +Where possible we recommend using a larger ASTC block size. This gives you +reliable savings to run-time memory footprint and memory bandwidth, in +addition to the packging size reduction. + +We recommend using RDO to fine-tune packaging size when dropping to the next +lowest ASTC bitrate is not an option. This could be because the next bitrate +option loses too much detail, or suffers from more visible block artifacts. + +## Building with RDO + +RDO support in the codec is disabled by default. Enable it using the +`ASTCENC_RDO` CMake option. For example, from the repository root on an +x86 machine supporting AVX2: + +```shell +cmake -S . -B build-rdo -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=clang++ -DASTCENC_ISA_AVX2=ON -DASTCENC_RDO=ON + +cmake --build build-rdo --config Release --parallel +``` + +> [!NOTE] +> See [Building ASTC Encoder](Building.md) for compiler requirements and other +> build configurations. + +On Linux, optionally add `-DASTCENC_RDO_SIZE=ON` when configuring to report the +serialized file size and its DEFLATE-compressed size. This requires the zlib +development package to be installed in your system: + +```shell +sudo apt-get install zlib1g-dev +``` + +Reporting the saving using DEFLATE does not actually compress the output file; +your packaging pipeline must still perform lossless compression. + +## Compressing with RDO + +Use the `-rdo-lambda` command line option to enable RDO and set its strength. +Setting 0.0 disables RDO, higher values increase the amount of quality loss +permitted. + +```shell +# Reference encode without RDO +build-rdo/Source/astcenc-avx2 -cl input.png reference.astc 5x5 -thorough \ + -rdo-lambda 0 + +# Same footprint and search preset, with RDO enabled +build-rdo/Source/astcenc-avx2 -cl input.png rdo.astc 5x5 -thorough \ + -rdo-lambda 0.2 +``` + +The RDO lambda value is a heuristic to control tradeoffs, not a specific target +bitrate or PSNR threshold. Lower positive values favor fidelity; higher values +place more weight on storage savings. A useful initial experiment is to compare +0.0, 0.1, 0.2, and 0.5 at the same block footprint and quality preset. Measure +quality and final packaged size for your own content. + +## Morton order file layout + +Ordinary `.astc` files, and other formats such as KTX, store ASTC blocks in +scanline order. Nearby rows in the image, that are likely to have more similar +colors, can therefore be far apart in the byte stream. + +The `.astcm` file format is a variant of the `.astc` format that stores the +astc blocks in the image as a sequence of 2D "tiles" of ASTC blocks arranged +in Morton order. This groups spatial neighbors together, which improves the +efficiency of the RDO data conditioning and the effectiveness of the +packaging-level compression. + +Select Morton order by writing to an `.astcm` file: + +```shell +build-rdo/Source/astcenc-avx2 -cl input.png rdo.astcm 5x5 -thorough \ + -rdo-lambda 0.2 +``` + +The `.astcm` file layout stores 8x8 tiles of ASTC blocks, with tiles being +stored sequentially in scanline order. Blocks within each tile are stored in +Morton order. Partial edge tiles omit blocks that fall outside the image. + +> [!NOTE] +> See [File formats](FileFormat.md#the-astcm-format) for the header and exact +> block traversal. + +### Data loading + +The `.astcm` data ordering is a storage layout that improves packing +efficiency, but it is not the layout that the graphcis APs expect. +Before you can upload the data to OpenGL ES or Vulkan you must convert +the data back into scanline block order: + +### Morton order in the library API + +When passing `ASTCENC_FLG_USE_MORTON` to the library API, it will issue tasks +to the compressor in blocks of one or more 8x8 Morton-order "tiles". This +allows the RDO transform to exploit 2D similarity in the image. + +However, it is important to note that the compressed ASTC data returned by the +codec API will be returned in block-by-block scanline order. Tool file +serialization must also apply the 8x8 Morton transform to realize the packaging +benefits. If you are not using Morton-order storage files, it is recommended +that you do not pass `ASTCENC_FLG_USE_MORTON` to the library API, + +## RDO Entropy Reduction Transform + +The ASTC Entroy Reduction Transform (AERT) used in our RDO implementation is +based on the public-domain Entropy Reduction Transform (ERT) implementation +developed by Rich Geldrich for [BC7Enc RDO][1]. The underlying algorithm is +the same, but it has been heavily optimized for ASTC to improve RDO +processing performance. + +- - - + +_Copyright © 2026, Arm Limited and contributors._ + +[1]: https://github.com/richgel999/bc7enc_rdo/blob/master/ert.h diff --git a/README.md b/README.md index 3ef0199b3..9f2e427e1 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,11 @@ ISA, we ship: * `astcenc-sse2` - uses SSE2 * `astcenc-sse4.1` - uses SSE4.1 and POPCNT -* `astcenc-avx2` - uses AVX2, SSE4.2, POPCNT, and F16C +* `astcenc-avx2` - uses AVX2, SSE4.2, POPCNT, F16C, BMI and BMI2. * `astcenc-avx256` - uses AVX-512F, AVX-512BW, AVX-512DQ, AVX-512VL, - AVX-512VBMI, POPCNT, and F16C + AVX-512VBMI, POPCNT, F16C, BMI and BMI2. * `astcenc-avx512` - uses AVX-512F, AVX-512BW, AVX-512DQ, AVX-512VL, - AVX-512VBMI, POPCNT, and F16C + AVX-512VBMI, POPCNT, F16C, BMI and BMI2. The x86-64 SSE2 builds will work on all x86-64 machines, but it is the slowest of the set. The other builds require extended CPU instruction set support which @@ -224,6 +224,7 @@ It covers: * How to efficiently encode data with fewer than 4 channels. * How to efficiently encode normal maps, sRGB data, and HDR data. * Coding equivalents to other compression formats. +* [Using rate-distortion optimization](./Docs/RDO.md). The [ASTC Developer Guide][5] document (external link) provides a more detailed guide for developers using the `astcenc` compressor. diff --git a/Source/UnitTest/cmake_core.cmake b/Source/UnitTest/cmake_core.cmake index c89bb8081..babb2f1e4 100644 --- a/Source/UnitTest/cmake_core.cmake +++ b/Source/UnitTest/cmake_core.cmake @@ -234,7 +234,7 @@ elseif(${ASTCENC_ISA_SIMD} MATCHES "avx512|avx256") target_compile_options(${ASTCENC_TEST} PRIVATE $<$:-mprefer-vector-width=256> - -mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mpopcnt -mf16c) + -mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mbmi -mbmi2 -mpopcnt -mf16c) elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") target_compile_definitions(${ASTCENC_TEST} @@ -248,7 +248,7 @@ elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") target_compile_options(${ASTCENC_TEST} PRIVATE - $<$>:-mavx2 -mpopcnt -mf16c> + $<$>:-mavx2 -mbmi -mbmi2 -mpopcnt -mf16c> $<$:/arch:AVX2>) endif() diff --git a/Source/UnitTest/test_decode.cpp b/Source/UnitTest/test_decode.cpp index b5fdc7750..4d587c9e4 100644 --- a/Source/UnitTest/test_decode.cpp +++ b/Source/UnitTest/test_decode.cpp @@ -29,9 +29,54 @@ #include "../astcenc.h" +#if defined(ASTCENC_RDO) +#include "../astcenc_internal.h" +#include "../astcenc_rdo.h" +#endif + namespace astcenc { +#if defined(ASTCENC_RDO) +/** @brief Every fixed QUANT decoder must preserve values, partial groups, and output bounds. */ +TEST(decompress, fixed_quantization_decode) +{ + for (unsigned int level = QUANT_2; level <= QUANT_256; level++) + { + auto quant = static_cast(level); + unsigned int alphabet_size = get_quant_level(quant); + rdo_ise_decode_func* fixed_decode = rdo_get_ise_decoder(level); + for (unsigned int count = 1; count <= 64; count++) + { + for (unsigned int offset = 0; offset < 8; offset++) + { + uint8_t input[64]; + uint8_t encoded[80] {}; + uint8_t decoded[66]; + uint8_t generic[64]; + std::fill_n(decoded, 66, 0xCD); + for (unsigned int i = 0; i < count; i++) + { + input[i] = static_cast((37 * i + 13 * count + offset) % alphabet_size); + } + + encode_ise(quant, count, input, encoded, offset); + fixed_decode(count, encoded, decoded + 1, offset); + decode_ise(quant, count, encoded, generic, offset); + for (unsigned int i = 0; i < count; i++) + { + ASSERT_EQ(decoded[i + 1], input[i]); + ASSERT_EQ(decoded[i + 1], generic[i]); + } + + EXPECT_EQ(decoded[0], 0xCD); + EXPECT_EQ(decoded[count + 1], 0xCD); + } + } + } +} +#endif + /** @brief Input overflow tests for xy * z. */ TEST(decompress, overflow_in_z) { diff --git a/Source/UnitTest/test_encode.cpp b/Source/UnitTest/test_encode.cpp index f89d60e63..273cb87b2 100644 --- a/Source/UnitTest/test_encode.cpp +++ b/Source/UnitTest/test_encode.cpp @@ -111,8 +111,10 @@ TEST(compress, mean_color_fallback) load_image_block(profile, image, blk, *bsd, 0, 0, 0, swizzle); // Grayscale with constant alpha skips the dual-plane search + symbolic_compressed_block compressed; + compress_block(ctx, blk, compressed, *working_buffers); uint8_t physical[16]; - compress_block(ctx, blk, physical, *working_buffers); + symbolic_to_physical(*bsd, compressed, physical); symbolic_compressed_block symbolic; physical_to_symbolic(*bsd, physical, symbolic); EXPECT_EQ(symbolic.block_type, expected_type); @@ -657,4 +659,55 @@ TEST(compress, data_nan_hdr_ldra) } } +#if defined(ASTCENC_RDO) +/** @brief RDO defaults to zero lambda and accepts nonnegative weights. */ +TEST(compress, rdo_configuration) +{ + astcenc_config config; + astcenc_error status = astcenc_config_init(ASTCENC_PRF_LDR, 4, 4, 1, + ASTCENC_PRE_MEDIUM, 0, &config); + ASSERT_EQ(status, ASTCENC_SUCCESS); + EXPECT_EQ(config.rdo_lambda, 0.0f); + + const float lambdas[] { 0.0f, 0.1f, 0.2f }; + for (float lambda : lambdas) + { + config.rdo_lambda = lambda; + astcenc_context* context = nullptr; + status = astcenc_context_alloc(&config, 1, &context, nullptr); + EXPECT_EQ(status, ASTCENC_SUCCESS); + if (context) + { + astcenc_context_free(context); + } + } +} + +/** @brief Non-finite and negative RDO lambdas are rejected. */ +TEST(compress, invalid_rdo_configuration) +{ + astcenc_config config; + astcenc_error status = astcenc_config_init(ASTCENC_PRF_LDR, 4, 4, 1, + ASTCENC_PRE_MEDIUM, 0, &config); + ASSERT_EQ(status, ASTCENC_SUCCESS); + astcenc_context* context = nullptr; + const float invalid_lambdas[] { + -1.0f, + std::numeric_limits::infinity(), + std::numeric_limits::quiet_NaN() + }; + for (float lambda : invalid_lambdas) + { + config.rdo_lambda = lambda; + status = astcenc_context_alloc(&config, 1, &context, nullptr); + EXPECT_EQ(status, ASTCENC_ERR_BAD_PARAM); + } + + config.rdo_lambda = 0.0f; + status = astcenc_context_alloc(&config, 1, &context, nullptr); + EXPECT_EQ(status, ASTCENC_SUCCESS); + astcenc_context_free(context); +} +#endif + } diff --git a/Source/UnitTest/test_image.cpp b/Source/UnitTest/test_image.cpp index 3e2a7ee53..a6bd2ab43 100644 --- a/Source/UnitTest/test_image.cpp +++ b/Source/UnitTest/test_image.cpp @@ -16,18 +16,20 @@ // ---------------------------------------------------------------------------- /** - * @brief Unit tests for strided image storage. + * @brief Unit tests for image storage and block traversal. */ #include #include #include +#include #include #include #include "gtest/gtest.h" #include "../astcenc.h" +#include "../astcenc_morton.h" namespace astcenc { @@ -215,6 +217,189 @@ TEST(image, row_stride) } } +/** + * @brief Check partial brick traversal preserves raster output without padding blocks. + * + * @param is_morton Whether to use Morton order instead of scanline order. + */ +static void test_bricks(bool is_morton) +{ + const unsigned int block_sizes[][3] { { 4, 4, 1 }, { 10, 6, 1 }, { 3, 3, 3 } }; + const astcenc_swizzle swizzle { + ASTCENC_SWZ_R, ASTCENC_SWZ_G, ASTCENC_SWZ_B, ASTCENC_SWZ_A + }; + const uint8_t constant_header[8] { 0xFC, 0xFD, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; + + for (const auto& block : block_sizes) + { + size_t brick_width = 128; + if (is_morton) + { + brick_width = 16; + } + + size_t width = block[0] * brick_width + 1; + size_t height = block[1] * 8 + 1; + size_t depth = 1; + if (block[2] > 1) + { + depth = block[2] * 2 + 1; + } + + size_t blocks_x = (width + block[0] - 1) / block[0]; + size_t blocks_y = (height + block[1] - 1) / block[1]; + size_t blocks_z = (depth + block[2] - 1) / block[2]; + size_t block_count = blocks_x * blocks_y * blocks_z; + size_t data_size = block_count * 16; + + // Each raster block has a distinct constant color, with guards outside the output payload + std::vector expected(data_size + 32, 0x55); + for (size_t i = 0; i < block_count; i++) + { + uint8_t* output = expected.data() + 16 + i * 16; + std::copy_n(constant_header, 8, output); + for (size_t channel = 0; channel < 4; channel++) + { + uint8_t value = static_cast(((i >> (channel * 3)) + channel * 47) & 0xFF); + output[8 + channel * 2] = value; + output[9 + channel * 2] = value; + } + } + + std::vector pixels(width * height * depth * 4); + std::vector slices(depth); + for (size_t z = 0; z < depth; z++) + { + slices[z] = pixels.data() + z * width * height * 4; + for (size_t y = 0; y < height; y++) + { + for (size_t x = 0; x < width; x++) + { + size_t index = ((z / block[2]) * blocks_y + y / block[1]) * blocks_x + + x / block[0]; + for (size_t channel = 0; channel < 4; channel++) + { + pixels[((z * height + y) * width + x) * 4 + channel] = + expected[16 + index * 16 + 8 + channel * 2]; + } + } + } + } + + astcenc_image image { + .dim_x = width, + .dim_y = height, + .dim_z = depth, + .data_type = ASTCENC_TYPE_U8, + .data = slices.data(), + .row_stride = 0 + }; + + unsigned int flags = 0; + if (is_morton) + { + flags |= ASTCENC_FLG_USE_MORTON; + } + + astcenc_config config; + astcenc_error status = astcenc_config_init( + ASTCENC_PRF_LDR, block[0], block[1], block[2], ASTCENC_PRE_FASTEST, flags, &config); + ASSERT_EQ(status, ASTCENC_SUCCESS); + + for (unsigned int thread_count : { 1u, 4u }) + { + SCOPED_TRACE(::testing::Message() << block[0] << 'x' << block[1] << 'x' << block[2] + << ", threads " << thread_count); + astcenc_context* context_raw; + status = astcenc_context_alloc(&config, thread_count, &context_raw, nullptr); + ASSERT_EQ(status, ASTCENC_SUCCESS); + astcenc_context_unq_ptr context(context_raw, astcenc_context_free); + + std::vector output(data_size + 32, 0x55); + std::vector results(thread_count); + std::vector workers; + for (unsigned int thread_index = 0; thread_index < thread_count; thread_index++) + { + workers.emplace_back([&, thread_index] { + results[thread_index] = astcenc_compress_image( + context.get(), &image, &swizzle, + output.data() + 16, data_size, thread_index); + }); + } + + for (auto& worker : workers) + { + worker.join(); + } + + for (astcenc_error result : results) + { + ASSERT_EQ(result, ASTCENC_SUCCESS); + } + + EXPECT_EQ(output, expected); + } + } +} + +/** @brief Morton bricks cover partial edges and preserve raster output. */ +TEST(image, morton_bricks) +{ + test_bricks(true); +} + +/** @brief Scanline bricks cover partial edges and preserve raster output. */ +TEST(image, scanline_bricks) +{ + test_bricks(false); +} + +/** @brief Both traversal orders cover their brick once, in the expected block order. */ +TEST(image, brick_layout) +{ + for (bool is_morton : { false, true }) + { + brick_layout layout(is_morton); + unsigned int expected_width = 128; + unsigned int expected_height = 1; + if (is_morton) + { + expected_width = 16; + expected_height = 8; + } + + EXPECT_EQ(layout.width, expected_width); + EXPECT_EQ(layout.height, expected_height); + brick_layout::validity_mask visited; + for (unsigned int index = 0; index < brick_layout::block_count; index++) + { + auto position = layout.get_position(index); + ASSERT_LT(position.x, expected_width); + ASSERT_LT(position.y, expected_height); + EXPECT_EQ(position.z, 0u); + size_t raster_index = position.y * expected_width + position.x; + bool is_visited = visited[raster_index]; + EXPECT_FALSE(is_visited); + visited.set(raster_index); + + size_t expected_index = position.x; + if (is_morton) + { + size_t x = position.x; + size_t y = position.y; + expected_index = (x & 1) | ((y & 1) << 1) | ((x & 2) << 1) | + ((y & 2) << 2) | ((x & 4) << 2) | ((y & 4) << 3) | + ((x & 8) << 3); + } + + EXPECT_EQ(index, expected_index); + } + + bool is_complete = visited.all(); + EXPECT_TRUE(is_complete); + } +} + /** @brief Invalid strides must be rejected before accessing image storage. */ TEST(image, invalid_row_stride) { diff --git a/Source/UnitTest/test_simd.cpp b/Source/UnitTest/test_simd.cpp index c464a0197..8beca9be5 100644 --- a/Source/UnitTest/test_simd.cpp +++ b/Source/UnitTest/test_simd.cpp @@ -25,6 +25,7 @@ * failing and the values in the test may need updating. */ +#include #include #include #include @@ -37,6 +38,189 @@ namespace astcenc { +/** @brief Integer vector arithmetic must wrap modulo 2^32 on every backend. */ +TEST(SuiteVint4, WrappingArithmetic) +{ + vint4 highest(std::numeric_limits::max()); + vint4 lowest(std::numeric_limits::min()); + + vint4 sum = highest + vint4(1); + vint4 difference = lowest - vint4(1); + vint4 product = highest * vint4(2); + + bool is_sum_valid = all(sum == lowest); + bool is_difference_valid = all(difference == highest); + bool is_product_valid = all(product == vint4(-2)); + + EXPECT_TRUE(is_sum_valid); + EXPECT_TRUE(is_difference_valid); + EXPECT_TRUE(is_product_valid); + + vint4 scalar_sum = highest + 1; + vint4 scalar_difference = lowest - 1; + vint4 scalar_product = highest * 2; + vint4 accumulator = highest; + + accumulator += vint4(1); + + bool is_scalar_sum_valid = all(scalar_sum == lowest); + bool is_scalar_difference_valid = all(scalar_difference == highest); + bool is_scalar_product_valid = all(scalar_product == vint4(-2)); + bool is_accumulator_valid = all(accumulator == lowest); + + EXPECT_TRUE(is_scalar_sum_valid); + EXPECT_TRUE(is_scalar_difference_valid); + EXPECT_TRUE(is_scalar_product_valid); + EXPECT_TRUE(is_accumulator_valid); +} + +/** @brief Active-width integer vector arithmetic must wrap modulo 2^32. */ +TEST(SuiteVint, WrappingArithmetic) +{ + vint highest(std::numeric_limits::max()); + vint lowest(std::numeric_limits::min()); + + vint sum = highest + vint(1); + vint difference = lowest - vint(1); + vint product = highest * vint(2); + vint accumulator = highest; + + accumulator += vint(1); + + bool is_sum_valid = all(sum == lowest); + bool is_difference_valid = all(difference == highest); + bool is_product_valid = all(product == vint(-2)); + bool is_accumulator_valid = all(accumulator == lowest); + + EXPECT_TRUE(is_sum_valid); + EXPECT_TRUE(is_difference_valid); + EXPECT_TRUE(is_product_valid); + EXPECT_TRUE(is_accumulator_valid); +} + +/** @brief Integer RGB sums must wrap modulo 2^32 and exclude alpha. */ +TEST(SuiteVint4, WrappingHorizontalSum) +{ + int highest = std::numeric_limits::max(); + int lowest = std::numeric_limits::min(); + + int positive_sum = hadd_rgb_s(vint4(highest, 1, 0, highest)); + int negative_sum = hadd_rgb_s(vint4(lowest, -1, 0, lowest)); + int intermediate_sum = hadd_rgb_s(vint4(highest, highest, 2, highest)); + + EXPECT_EQ(positive_sum, lowest); + EXPECT_EQ(negative_sum, highest); + EXPECT_EQ(intermediate_sum, 0); +} + +/** @brief Packed bit operations must handle empty, sparse, and full-width fields. */ +TEST(SuiteMisc, PackedBitOperations) +{ + uint64_t empty = mask_low_bits(~0ull, 0); + uint64_t full = mask_low_bits(~0ull, 64); + uint64_t high = extract_bit_range(1ull << 63, 63, 1); + uint64_t compressed = compress_bits(0xA55A, 0x0F0F); + uint64_t deposited = deposit_bits(0x5A, 0x0F0F); + uint64_t all_compressed = compress_bits(0x123456789ABCDEF0ull, ~0ull); + uint64_t all_deposited = deposit_bits(0x123456789ABCDEF0ull, ~0ull); + EXPECT_EQ(empty, 0u); + EXPECT_EQ(full, ~0ull); + EXPECT_EQ(high, 1u); + EXPECT_EQ(compressed, 0x5Au); + EXPECT_EQ(deposited, 0x050Au); + EXPECT_EQ(all_compressed, 0x123456789ABCDEF0ull); + EXPECT_EQ(all_deposited, 0x123456789ABCDEF0ull); +} + +/** @brief Fixed-width bit fields and byte lookups must preserve packed byte order. */ +TEST(SuiteVint4, PackedByteOperations) +{ + uint64_t packed = 0xFEDCBA9876543210ull; + for (unsigned int bits = 1; bits <= 8; bits++) + { + uint8_t bytes[16]; + vint4 fields = unpack_byte_fields(packed, bits); + store(fields, bytes); + for (unsigned int i = 0; i < 8; i++) + { + uint8_t expected = static_cast((packed >> (i * bits)) & ((1u << bits) - 1)); + EXPECT_EQ(bytes[i], expected); + EXPECT_EQ(bytes[8 + i], 0); + } + } + + uint8_t table_bytes[32]; + for (unsigned int i = 0; i < 32; i++) + { + table_bytes[i] = static_cast(i * 7 + 3); + } + + vtable4_16x8 table16; + vtable4_32x8 table32; + vtable_prepare(table16, table_bytes); + vtable_prepare(table32, table_bytes); + for (unsigned int half = 0; half < 2; half++) + { + uint8_t indices[16]; + for (unsigned int i = 0; i < 16; i++) + { + indices[i] = static_cast(2 * i + half); + } + + vint4 input = vint4::load(indices); + vint4 values = vtable_lookup_8bit(table32, input); + uint8_t bytes[16]; + store(values, bytes); + for (unsigned int i = 0; i < 16; i++) + { + EXPECT_EQ(bytes[i], table_bytes[indices[i]]); + indices[i] &= 15; + } + + input = vint4::load(indices); + values = vtable_lookup_8bit(table16, input); + store(values, bytes); + for (unsigned int i = 0; i < 16; i++) + { + EXPECT_EQ(bytes[i], table_bytes[indices[i]]); + } + } +} + +/** @brief Packed byte stores must deinterleave correctly and write exactly eight bytes. */ +TEST(SuiteVint4, PackedByteStores) +{ + uint8_t bytes[16]; + uint8_t even[10]; + uint8_t odd[10]; + uint8_t low[10]; + std::fill_n(even, 10, 0xFE); + std::fill_n(odd, 10, 0xFE); + std::fill_n(low, 10, 0xFE); + for (unsigned int i = 0; i < 16; i++) + { + bytes[i] = static_cast(i); + } + + vint4 input = vint4::load(bytes); + store_deinterleaved_bytes(input, even + 1, odd + 1); + store_low_8bytes(input, low + 1); + for (unsigned int i = 0; i < 8; i++) + { + EXPECT_EQ(even[i + 1], 2 * i); + EXPECT_EQ(odd[i + 1], 2 * i + 1); + EXPECT_EQ(low[i + 1], i); + } + + EXPECT_EQ(even[0], 0xFE); + EXPECT_EQ(even[9], 0xFE); + EXPECT_EQ(odd[0], 0xFE); + EXPECT_EQ(odd[9], 0xFE); + EXPECT_EQ(low[0], 0xFE); + EXPECT_EQ(low[9], 0xFE); +} + + // Misc utility tests - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #if ASTCENC_SIMD_WIDTH == 8 /** diff --git a/Source/astcenc.h b/Source/astcenc.h index 282dc004f..c1529ed64 100644 --- a/Source/astcenc.h +++ b/Source/astcenc.h @@ -405,6 +405,14 @@ static const unsigned int ASTCENC_FLG_SELF_DECOMPRESS_ONLY = 1 << 5; */ static const unsigned int ASTCENC_FLG_MAP_RGBM = 1 << 6; +/** + * @brief Use 16x8 Morton-ordered encoder bricks instead of 128x1 scanline bricks. + * + * RDO optimizes blocks in the selected brick order. The compressed image buffer remains in raster + * order; callers storing Morton-ordered files must reorder the blocks during serialization. + */ +static const unsigned int ASTCENC_FLG_USE_MORTON = 1 << 7; + /** * @brief The bit mask of all valid flags. */ @@ -415,7 +423,8 @@ static const unsigned int ASTCENC_ALL_FLAGS = ASTCENC_FLG_USE_PERCEPTUAL | ASTCENC_FLG_USE_DECODE_UNORM8 | ASTCENC_FLG_DECOMPRESS_ONLY | - ASTCENC_FLG_SELF_DECOMPRESS_ONLY; + ASTCENC_FLG_SELF_DECOMPRESS_ONLY | + ASTCENC_FLG_USE_MORTON; /** * @brief The config structure. @@ -606,6 +615,15 @@ struct astcenc_config */ astcenc_progress_callback progress_callback; +#if defined(ASTCENC_RDO) + /** + * @brief The experimental RDO rate weight; zero disables RDO and is the default. + * + * Requires ASTCENC_RDO to be defined for both the library and its callers. + */ + float rdo_lambda; +#endif + #if defined(ASTCENC_DIAGNOSTICS) /** * @brief The path to save the diagnostic trace data to. diff --git a/Source/astcenc_compress_symbolic.cpp b/Source/astcenc_compress_symbolic.cpp index 4558393eb..3eb7a9ec1 100644 --- a/Source/astcenc_compress_symbolic.cpp +++ b/Source/astcenc_compress_symbolic.cpp @@ -1530,11 +1530,10 @@ static float prepare_block_statistics( void compress_block( const astcenc_contexti& ctx, const image_block& blk, - uint8_t pcb[16], + symbolic_compressed_block& scb, compression_working_buffers& tmpbuf) { astcenc_profile decode_mode = ctx.config.profile; - symbolic_compressed_block scb; const block_size_descriptor& bsd = *ctx.bsd; float lowest_correl = 0.0f; unsigned int plane_components[BLOCK_MAX_COMPONENTS] { 3, 2, 1, 0 }; @@ -1611,7 +1610,6 @@ void compress_block( trace_add_data("exit", "quality hit"); - symbolic_to_physical(bsd, scb, pcb); return; } @@ -1856,9 +1854,6 @@ void compress_block( store(color_u16, scb.constant_color); } } - - // Compress to a physical block - symbolic_to_physical(bsd, scb, pcb); } #endif diff --git a/Source/astcenc_entry.cpp b/Source/astcenc_entry.cpp index b88bb9ed2..da88b99a3 100644 --- a/Source/astcenc_entry.cpp +++ b/Source/astcenc_entry.cpp @@ -20,7 +20,7 @@ */ #include -#include +#include #include #include @@ -28,8 +28,13 @@ #include "astcenc_diagnostic_trace.h" #include "astcenc_internal_entry.h" #include "astcenc_mathlib.h" +#include "astcenc_morton.h" #include "astcenc_preset_configs.h" +#if defined(ASTCENC_RDO) +#include "astcenc_rdo.h" +#endif + /** * @brief Validate image storage and resolve the default row stride. * @@ -379,6 +384,13 @@ static astcenc_error validate_config( return status; } +#if defined(ASTCENC_RDO) + if (!std::isfinite(config.rdo_lambda) || (config.rdo_lambda < 0.0f)) + { + return ASTCENC_ERR_BAD_PARAM; + } +#endif + // A block_z of 0 is accepted to mean 2D, normalize to 1 to avoid div-by-zero later config.block_z = astc::max(config.block_z, 1u); @@ -472,6 +484,9 @@ astcenc_error astcenc_config_init( config.block_x = block_x; config.block_y = block_y; config.block_z = block_z; +#if defined(ASTCENC_RDO) + config.rdo_lambda = 0.0f; +#endif float texels = static_cast(block_x * block_y * block_z); float ltexels = logf(texels) / logf(10.0f); @@ -758,6 +773,7 @@ astcenc_error astcenc_context_alloc( } bool can_omit_modes = static_cast(config.flags & ASTCENC_FLG_SELF_DECOMPRESS_ONLY); + init_block_size_descriptor(config.block_x, config.block_y, config.block_z, can_omit_modes, config.tune_partition_count_limit, @@ -850,13 +866,224 @@ void astcenc_context_free( #if !defined(ASTCENC_DECOMPRESS_ONLY) +/** @brief A sparse brick of symbolic blocks indexed in the selected traversal order. */ +struct compression_brick +{ + /** @brief A bit for each in-range brick position; other slots must not be read. */ + brick_layout::validity_mask valid_mask; + + /** @brief The symbolic encoding at each valid brick position. */ + symbolic_compressed_block blocks[brick_layout::block_count]; + + /** @brief The raster output byte offset at each valid brick position. */ + size_t offsets[brick_layout::block_count]; +}; + +/** + * @brief Load and compress one selected image block to symbolic form. + * + * @param[in] ctx The compressor context. + * @param[in] image The input image. + * @param[in] swizzle The input swizzle. + * @param[in] load_func The image block loader selected for this image. + * @param[in] channel_weight The configured weights before per-block adjustments. + * @param[in] origin The block origin in texels. + * @param[in,out] blk The reusable image block storage. + * @param[in,out] temp_buffers The thread's compression scratch buffers. + * @param[out] scb The symbolic encoding of this block. + */ +static void compress_image_block( + const astcenc_contexti& ctx, + const astcenc_image& image, + const astcenc_swizzle& swizzle, + decltype(&load_image_block) load_func, + const vfloat4& channel_weight, + const position& origin, + image_block& blk, + compression_working_buffers& temp_buffers, + symbolic_compressed_block& scb +) { + const block_size_descriptor& bsd = *ctx.bsd; + astcenc_profile decode_mode = ctx.config.profile; + size_t block_x = bsd.dim_x; + size_t block_y = bsd.dim_y; + size_t block_z = bsd.dim_z; + size_t dim_x = image.dim_x; + size_t dim_y = image.dim_y; + bool is_hdr = (decode_mode == ASTCENC_PRF_HDR) || + (decode_mode == ASTCENC_PRF_HDR_RGB_LDR_A); + + // Check for blocks with negligible alpha contribution + bool is_full_block_needed = true; + if ((ctx.config.a_scale_radius != 0) && (block_z == 1)) + { + size_t start_x = origin.x; + size_t end_x = astc::min(dim_x, start_x + block_x); + + size_t start_y = origin.y; + size_t end_y = astc::min(dim_y, start_y + block_y); + + // SATs accumulate error, so don't test exactly zero. Test for less than 1 alpha + // in the expanded block footprint that includes the alpha radius. + size_t x_footprint = block_x + 2 * (ctx.config.a_scale_radius - 1); + size_t y_footprint = block_y + 2 * (ctx.config.a_scale_radius - 1); + + float footprint = static_cast(x_footprint * y_footprint); + float threshold = 0.9f / (255.0f * footprint); + + // Check for alpha values above the threshold + is_full_block_needed = false; + for (size_t ay = start_y; ay < end_y; ay++) + { + for (size_t ax = start_x; ax < end_x; ax++) + { + float a_avg = ctx.input_alpha_averages[ay * dim_x + ax]; + if (a_avg > threshold) + { + is_full_block_needed = true; + ax = end_x; + ay = end_y; + } + } + } + } + + // Fetch the full block for compression + blk.channel_weight = channel_weight; + if (is_full_block_needed) + { + load_func(decode_mode, image, blk, bsd, origin.x, origin.y, origin.z, swizzle); + + if (is_hdr && (ctx.config.flags & ASTCENC_FLG_USE_PERCEPTUAL)) + { + // Approximate display-space error by weighting logarithmic channels by + // their geometric brightness + vfloat4 mean = blk.data_mean; + mean.set_lane<3>(0.0f); + vfloat4 geometric_mean = float16_to_float(lns_to_sf16(float_to_int(mean))); + float maximum = astc::max(hmax_s(geometric_mean), 1e-20f); + + // Retain at least 1% weight for dim channels + vfloat4 weights = max(geometric_mean / maximum, vfloat4(0.01f)); + weights.set_lane<3>(1.0f); + blk.channel_weight = weights * blk.channel_weight; + } + + // Scale RGB error contribution by the maximum alpha in the block. This + // encourages alpha accuracy in transparent regions, gaining up to 0.5 dB. + if (ctx.config.flags & ASTCENC_FLG_USE_ALPHA_WEIGHT) + { + float alpha_scale = blk.data_max.lane<3>() * (1.0f / 65535.0f); + if (decode_mode == ASTCENC_PRF_HDR) + { + // Transparency uses linear alpha, even when the endpoints use LNS + vint4 alpha_lns = float_to_int(blk.data_max); + vfloat4 alpha = float16_to_float(lns_to_sf16(alpha_lns)); + alpha_scale = astc::clamp1f(alpha.lane<3>()); + } + + blk.channel_weight = blk.channel_weight + * vfloat4(alpha_scale, alpha_scale, alpha_scale, 1.0f); + } + } + // Apply alpha scale RDO - substitute constant color block + else + { + blk.origin_texel = vfloat4::zero(); + blk.data_min = vfloat4::zero(); + blk.data_mean = vfloat4::zero(); + blk.data_max = vfloat4::zero(); + blk.grayscale = true; + } + + compress_block(ctx, blk, scb, temp_buffers); +} + +/** + * @brief Compress every in-range block in a brick to symbolic form. + * + * @param[in] ctx The compressor context. + * @param[in] image The input image. + * @param[in] swizzle The input swizzle. + * @param[in] load_func The image block loader selected for this image. + * @param[in] channel_weight The configured weights before per-block adjustments. + * @param[in] origin The brick origin in ASTC blocks. + * @param[in,out] blk The reusable image block storage. + * @param[in,out] temp_buffers The thread's compression scratch buffers. + * @param[out] brick The sparse symbolic encodings and raster output offsets. + * @param[in] layout The brick dimensions and traversal order. + */ +static void brick_compress_to_symbolic( + const astcenc_contexti& ctx, + const astcenc_image& image, + const astcenc_swizzle& swizzle, + decltype(&load_image_block) load_func, + const vfloat4& channel_weight, + const position& origin, + image_block& blk, + compression_working_buffers& temp_buffers, + compression_brick& brick, + const brick_layout& layout +) { + const block_size_descriptor& bsd = *ctx.bsd; + size_t blocks_x = astc::get_block_count_safe(image.dim_x, bsd.dim_x); + size_t blocks_y = astc::get_block_count_safe(image.dim_y, bsd.dim_y); + + // Reused edge bricks may contain stale data outside the current valid mask + brick.valid_mask.reset(); + for (unsigned int index = 0; index < brick_layout::block_count; index++) + { + auto local_position = layout.get_position(index); + size_t x = origin.x + local_position.x; + size_t y = origin.y + local_position.y; + if ((x >= blocks_x) || (y >= blocks_y)) + { + continue; + } + + const position texel_position { + .x = x * bsd.dim_x, + .y = y * bsd.dim_y, + .z = origin.z * bsd.dim_z + }; + compress_image_block(ctx, image, swizzle, load_func, channel_weight, + texel_position, blk, temp_buffers, brick.blocks[index]); + + brick.offsets[index] = ((origin.z * blocks_y + y) * blocks_x + x) * 16; + brick.valid_mask.set(index); + } +} + +/** + * @brief Pack every valid symbolic block in a brick into the raster-ordered output stream. + * + * @param[in] bsd The block size descriptor. + * @param[in] brick The sparse symbolic encodings and raster output offsets. + * @param[out] buffer The output array for the compressed image. + */ +static void brick_symbolic_to_physical( + const block_size_descriptor& bsd, + const compression_brick& brick, + uint8_t* buffer +) { + for (unsigned int index = 0; index < brick_layout::block_count; index++) + { + if (!brick.valid_mask[index]) + { + continue; + } + + symbolic_to_physical(bsd, brick.blocks[index], buffer + brick.offsets[index]); + } +} + /** * @brief Compress an image, after any preflight has completed. * * @param[in,out] ctxo The compressor context. * @param thread_index The thread index. - * @param image The input image. - * @param swizzle The input swizzle. + * @param[in] image The input image. + * @param[in] swizzle The input swizzle. * @param[out] buffer The output array for the compressed data. */ static void compress_image( @@ -871,6 +1098,8 @@ static void compress_image( astcenc_profile decode_mode = ctx.config.profile; image_block blk {}; + compression_brick brick; + const brick_layout layout(ctx.config.flags & ASTCENC_FLG_USE_MORTON); size_t block_x = bsd.dim_x; size_t block_y = bsd.dim_y; @@ -885,18 +1114,18 @@ static void compress_image( size_t blocks_y = astc::get_block_count_safe(dim_y, block_y); size_t blocks_z = astc::get_block_count_safe(dim_z, block_z); - size_t block_count = get_blocks_count(blocks_x, blocks_y, blocks_z); - // Should never fail here - tested in caller before calling here - assert(block_count > 0); - - - size_t row_blocks = blocks_x; - size_t plane_blocks = blocks_x * blocks_y; + // The caller has validated the block grid, so the smaller brick grid also fits in size_t + size_t bricks_x = astc::get_block_count_safe(blocks_x, layout.width); + size_t bricks_y = astc::get_block_count_safe(blocks_y, layout.height); + size_t plane_bricks = bricks_x * bricks_y; + size_t brick_count = plane_bricks * blocks_z; + assert(brick_count > 0); blk.decode_unorm8 = ctxo.context.config.flags & ASTCENC_FLG_USE_DECODE_UNORM8; // The L+A refit skips the RGB+offset endpoint calculation needed for HDR mode 7 - bool is_ldr = ctx.config.profile == ASTCENC_PRF_LDR || ctx.config.profile == ASTCENC_PRF_LDR_SRGB; + bool is_ldr = (ctx.config.profile == ASTCENC_PRF_LDR) || + (ctx.config.profile == ASTCENC_PRF_LDR_SRGB); if (is_ldr) { blk.active_channel_mask = ctx.active_channel_mask; @@ -916,7 +1145,7 @@ static void compress_image( auto& temp_buffers = ctx.working_buffers[thread_index]; // Only the first thread runs initialization - ctxo.manage_compress.init(block_count, ctx.config.progress_callback); + ctxo.manage_compress.init(brick_count, ctx.config.progress_callback, brick_layout::block_count); // Select the optimized loader when supported bool needs_swz = (swizzle.r != ASTCENC_SWZ_R) || (swizzle.g != ASTCENC_SWZ_G) || @@ -938,7 +1167,7 @@ static void compress_image( while (true) { size_t count; - size_t base = ctxo.manage_compress.get_task_assignment(16, count); + size_t base = ctxo.manage_compress.get_task_assignment(1, count); if (!count) { break; @@ -946,98 +1175,27 @@ static void compress_image( for (size_t i = base; i < base + count; i++) { - // Decode i into x, y, z block indices - size_t z = i / plane_blocks; - size_t rem = i - (z * plane_blocks); - size_t y = rem / row_blocks; - size_t x = rem - (y * row_blocks); - - // Check for blocks with negligible alpha contribution - bool use_full_block = true; - if (ctx.config.a_scale_radius != 0 && block_z == 1) - { - size_t start_x = x * block_x; - size_t end_x = astc::min(dim_x, start_x + block_x); - - size_t start_y = y * block_y; - size_t end_y = astc::min(dim_y, start_y + block_y); - - // SATs accumulate error, so don't test exactly zero. Test for less than 1 alpha in - // the expanded block footprint that includes the alpha radius. - size_t x_footprint = block_x + 2 * (ctx.config.a_scale_radius - 1); - size_t y_footprint = block_y + 2 * (ctx.config.a_scale_radius - 1); - - float footprint = static_cast(x_footprint * y_footprint); - float threshold = 0.9f / (255.0f * footprint); - - // Check for alpha values above the threshold - use_full_block = false; - for (size_t ay = start_y; ay < end_y; ay++) - { - for (size_t ax = start_x; ax < end_x; ax++) - { - float a_avg = ctx.input_alpha_averages[ay * dim_x + ax]; - if (a_avg > threshold) - { - use_full_block = true; - ax = end_x; - ay = end_y; - } - } - } - } - - // Fetch the full block for compression - blk.channel_weight = channel_weight; - if (use_full_block) - { - load_func(decode_mode, image, blk, bsd, x * block_x, y * block_y, z * block_z, swizzle); - - if (is_hdr && (ctx.config.flags & ASTCENC_FLG_USE_PERCEPTUAL)) - { - // Approximate display-space error by weighting logarithmic channels by their - // geometric brightness - vfloat4 mean = blk.data_mean; - mean.set_lane<3>(0.0f); - vfloat4 geometric_mean = float16_to_float(lns_to_sf16(float_to_int(mean))); - float maximum = astc::max(hmax_s(geometric_mean), 1e-20f); - - // Retain at least 1% weight for dim channels - vfloat4 weights = max(geometric_mean / maximum, vfloat4(0.01f)); - weights.set_lane<3>(1.0f); - blk.channel_weight = weights * blk.channel_weight; - } - - // Scale RGB error contribution by the maximum alpha in the block. This encourages - // alpha accuracy in regions with high transparency, and can gain up to 0.5 dB. - if (ctx.config.flags & ASTCENC_FLG_USE_ALPHA_WEIGHT) - { - float alpha_scale = blk.data_max.lane<3>() * (1.0f / 65535.0f); - if (decode_mode == ASTCENC_PRF_HDR) - { - // Transparency uses linear alpha, even when the endpoints use LNS - vint4 alpha_lns = float_to_int(blk.data_max); - vfloat4 alpha = float16_to_float(lns_to_sf16(alpha_lns)); - alpha_scale = astc::clamp1f(alpha.lane<3>()); - } - - blk.channel_weight = blk.channel_weight - * vfloat4(alpha_scale, alpha_scale, alpha_scale, 1.0f); - } - } - // Apply alpha scale RDO - substitute constant color block - else - { - blk.origin_texel = vfloat4::zero(); - blk.data_min = vfloat4::zero(); - blk.data_mean = vfloat4::zero(); - blk.data_max = vfloat4::zero(); - blk.grayscale = true; - } - - size_t offset = ((z * blocks_y + y) * blocks_x + x) * 16; - uint8_t *bp = buffer + offset; - compress_block(ctx, blk, bp, temp_buffers); + // Each task owns one brick in a single block-depth plane + size_t z = i / plane_bricks; + size_t rem = i - (z * plane_bricks); + size_t brick_y = rem / bricks_x; + size_t brick_x = rem - (brick_y * bricks_x); + + const position origin { + .x = brick_x * layout.width, + .y = brick_y * layout.height, + .z = z + }; + + brick_compress_to_symbolic(ctx, image, swizzle, load_func, channel_weight, + origin, blk, temp_buffers, brick, layout); + + brick_symbolic_to_physical(bsd, brick, buffer); + +#if defined(ASTCENC_RDO) + rdo_optimize_physical_brick(ctx, image, swizzle, brick.valid_mask, + brick.offsets, brick.blocks, buffer); +#endif } ctxo.manage_compress.complete_task_assignment(count); diff --git a/Source/astcenc_integer_sequence.cpp b/Source/astcenc_integer_sequence.cpp index a4dc6511c..28a9aa024 100644 --- a/Source/astcenc_integer_sequence.cpp +++ b/Source/astcenc_integer_sequence.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // ---------------------------------------------------------------------------- -// Copyright 2011-2024 Arm Limited +// Copyright 2011-2026 Arm Limited // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy @@ -21,10 +21,15 @@ #include "astcenc_internal.h" +#if defined(ASTCENC_RDO) +#include "astcenc_rdo.h" +#endif + #include +#include /** @brief Unpacked quint triplets for each packed value. */ -static const uint8_t quints_of_integer[128][3] { +static constexpr uint8_t quints_of_integer[128][3] { {0, 0, 0}, {1, 0, 0}, {2, 0, 0}, {3, 0, 0}, {4, 0, 0}, {0, 4, 0}, {4, 4, 0}, {4, 4, 4}, {0, 1, 0}, {1, 1, 0}, {2, 1, 0}, {3, 1, 0}, @@ -99,7 +104,7 @@ static const uint8_t integer_of_quints[5][5][5] { }; /** @brief Unpacked trit quintuplets for each packed value. */ -static const uint8_t trits_of_integer[256][5] { +static constexpr uint8_t trits_of_integer[256][5] { {0, 0, 0, 0, 0}, {1, 0, 0, 0, 0}, {2, 0, 0, 0, 0}, {0, 0, 2, 0, 0}, {0, 1, 0, 0, 0}, {1, 1, 0, 0, 0}, {2, 1, 0, 0, 0}, {1, 0, 2, 0, 0}, {0, 2, 0, 0, 0}, {1, 2, 0, 0, 0}, {2, 2, 0, 0, 0}, {2, 0, 2, 0, 0}, @@ -473,7 +478,7 @@ static inline void write_bits( * * @return The read value. */ -static inline unsigned int read_bits( +[[maybe_unused]] static inline unsigned int read_bits( unsigned int bitcount, unsigned int bitoffset, const uint8_t* ptr @@ -487,6 +492,318 @@ static inline unsigned int read_bits( return value; } +/** + * @brief Pack radix digits into byte lanes for a single table load per ISE group. + * + * @tparam EntryCount The number of radix codes. + * @tparam ValueCount The number of digits per code. + * @param values The decoded radix digits. + * + * @return The digit groups packed into little-endian words. + */ +template +static constexpr std::array pack_radix_table( + const uint8_t (&values)[EntryCount][ValueCount] +) { + std::array packed {}; + for (size_t i = 0; i < EntryCount; i++) + { + for (size_t j = 0; j < ValueCount; j++) + { + packed[i] |= static_cast(values[i][j]) << (8 * j); + } + } + + return packed; +} + +/** @brief Three quint digits stored in byte lanes. */ +static constexpr auto packed_quints = pack_radix_table(quints_of_integer); + +/** @brief Five trit digits stored in byte lanes. */ +static constexpr auto packed_trits = pack_radix_table(trits_of_integer); + +#if defined(ASTCENC_RDO) +/* See astcenc_rdo.h for documentation. */ +void rdo_decode_ise_radix_weights( + unsigned int quant_level, + unsigned int count, + const uint8_t* input_data, + uint8_t* output_data +) { + if (quant_level == QUANT_3) + { + unsigned int i = 0; + for (; i + 5 <= count; i += 5) + { + uint64_t weights = packed_trits[input_data[i / 5]] << 5; + store_le64_prefix(weights, output_data + i, 5); + } + + unsigned int tail = count - i; + if (tail != 0) + { + unsigned int bits = (tail * 8 + 4) / 5; + unsigned int code = input_data[i / 5] & ((1u << bits) - 1); + uint64_t weights = packed_trits[code] << 5; + for (unsigned int j = 0; j < tail; j++) + { + output_data[i + j] = static_cast(weights >> (8 * j)); + } + } + } + else + { + assert(quant_level == QUANT_5); + unsigned int i = 0; + unsigned int offset = 0; + for (; i + 3 <= count; i += 3) + { + unsigned int code = read_bits(7, offset, input_data); + uint64_t weights = packed_quints[code] << 4; + store_le64_prefix(weights, output_data + i, 3); + offset += 7; + } + + unsigned int tail = count - i; + if (tail != 0) + { + unsigned int bits = (tail * 7 + 2) / 3; + unsigned int code = read_bits(bits, offset, input_data); + uint64_t weights = packed_quints[code] << 4; + for (unsigned int j = 0; j < tail; j++) + { + output_data[i + j] = static_cast(weights >> (8 * j)); + } + } + } +} +#endif + +/** + * @brief Read a little-endian ISE group without accessing bytes outside the group. + * + * @param input_data The packed sequence. + * @param bit_offset The initial bit offset in the packed sequence. + * @param bit_count The number of group bits, at most 56. + * + * @return The extracted group bits. + */ +static ASTCENC_SIMD_INLINE uint64_t read_ise_group_packed( + const uint8_t* input_data, + unsigned int bit_offset, + unsigned int bit_count +) { + unsigned int byte_count = ((bit_offset & 7) + bit_count + 7) / 8; + const uint8_t* source = input_data + (bit_offset >> 3); + uint64_t packed = 0; + unsigned int shift = 0; + if (byte_count >= 4) + { + uint32_t value; + std::memcpy(&value, source, 4); + packed = value; + source += 4; + byte_count -= 4; + shift = 32; + } + + if (byte_count >= 2) + { + uint16_t value; + std::memcpy(&value, source, 2); + packed |= static_cast(value) << shift; + source += 2; + byte_count -= 2; + shift += 16; + } + + if (byte_count != 0) + { + packed |= static_cast(*source) << shift; + } + + return extract_bit_range(packed, bit_offset & 7, bit_count); +} + +/** + * @brief Decode binary-only ISE values using packed byte-lane expansion. + * + * @param bits The number of bits per value, in the range 1..8. + * @param count The number of values to decode. + * @param input_data The packed sequence. + * @param[out] output_data The decoded values. + * @param bit_offset The initial bit offset in the packed sequence. + */ +[[maybe_unused]] static ASTCENC_SIMD_INLINE void decode_ise_bits_packed( + unsigned int bits, + unsigned int count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + // Seven eight-bit values still fit a word when the sequence starts between bytes + unsigned int group_size = 8; + if (bits == 8) + { + group_size = 7; + } + + uint64_t lane_mask = 0x0101010101010101ull * mask_low_bits(~0ull, bits); + unsigned int input_bytes = (bit_offset + count * bits + 7) / 8; + for (unsigned int i = 0; i < count; i += group_size) + { + unsigned int value_count = astc::min(group_size, count - i); + unsigned int bit_count = value_count * bits; + uint64_t packed; + if (input_bytes >= 8) + { + unsigned int byte_offset = astc::min(bit_offset >> 3, input_bytes - 8); + std::memcpy(&packed, input_data + byte_offset, 8); + packed = extract_bit_range(packed, bit_offset - 8 * byte_offset, bit_count); + } + else + { + packed = read_ise_group_packed(input_data, bit_offset, bit_count); + } + + uint64_t decoded = deposit_bits(packed, lane_mask); + if (value_count == 8) + { + std::memcpy(output_data + i, &decoded, 8); + } + else + { + for (unsigned int j = 0; j < value_count; j++) + { + output_data[i + j] = static_cast(decoded >> (8 * j)); + } + } + + bit_offset += bit_count; + } +} + +/** + * @brief Decode interleaved five-value trit groups using packed bit extraction. + * + * @tparam HasBits True for mixed binary and radix values. + * + * @param bits The number of low binary bits per value. + * @param count The number of values to decode. + * @param input_data The packed sequence. + * @param[out] output_data The decoded values. + * @param bit_offset The initial bit offset in the packed sequence. + */ +template +static ASTCENC_SIMD_INLINE void decode_ise_trits_packed( + unsigned int bits, + unsigned int count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + if constexpr (!HasBits) + { + bits = 0; + } + + // Low binary bits and shared radix-code bits occupy disjoint positions in each group + static constexpr uint64_t low_masks[7] { + 0x0000000000ull, 0x0000000949ull, 0x0000019B33ull, 0x000039DCE7ull, + 0x00079EF3CFull, 0x00F9F7CF9Full, 0x1F9FBF3F3Full + }; + uint64_t low_mask = low_masks[bits]; + uint64_t lane_mask = 0x0101010101ull * mask_low_bits(~0ull, bits); + uint64_t code_mask = mask_low_bits(~low_mask, 5 * bits + 8); + unsigned int full_count = (count / 5) * 5; + unsigned int bit_count = 5 * bits + 8; + unsigned int i = 0; + for (; i < full_count; i += 5) + { + uint64_t packed = read_ise_group_packed(input_data, bit_offset, bit_count); + unsigned int code = static_cast(compress_bits(packed, code_mask)); + uint64_t decoded = packed_trits[code]; + decoded = (decoded << bits) | deposit_bits(compress_bits(packed, low_mask), lane_mask); + std::memcpy(output_data + i, &decoded, 5); + bit_offset += bit_count; + } + + unsigned int tail_count = count - i; + if (tail_count != 0) + { + bit_count = tail_count * bits + (tail_count * 8 + 4) / 5; + uint64_t packed = read_ise_group_packed(input_data, bit_offset, bit_count); + unsigned int code = static_cast(compress_bits(packed, code_mask)); + uint64_t decoded = packed_trits[code]; + decoded = (decoded << bits) | deposit_bits(compress_bits(packed, low_mask), lane_mask); + for (unsigned int j = 0; j < tail_count; j++) + { + output_data[i + j] = static_cast(decoded >> (8 * j)); + } + } +} + +/** + * @brief Decode interleaved three-value quint groups using packed bit extraction. + * + * @tparam HasBits True for mixed binary and radix values. + * + * @param bits The number of low binary bits per value. + * @param count The number of values to decode. + * @param input_data The packed sequence. + * @param[out] output_data The decoded values. + * @param bit_offset The initial bit offset in the packed sequence. + */ +template +static ASTCENC_SIMD_INLINE void decode_ise_quints_packed( + unsigned int bits, + unsigned int count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + if constexpr (!HasBits) + { + bits = 0; + } + + // Low binary bits and shared radix-code bits occupy disjoint positions in each group + static constexpr uint64_t low_masks[6] { + 0x0000000000ull, 0x0000000091ull, 0x0000000663ull, + 0x00000039C7ull, 0x000001E78Full, 0x00000F9F1Full + }; + uint64_t low_mask = low_masks[bits]; + uint64_t lane_mask = 0x010101ull * mask_low_bits(~0ull, bits); + uint64_t code_mask = mask_low_bits(~low_mask, 3 * bits + 7); + unsigned int full_count = (count / 3) * 3; + unsigned int bit_count = 3 * bits + 7; + unsigned int i = 0; + for (; i < full_count; i += 3) + { + uint64_t packed = read_ise_group_packed(input_data, bit_offset, bit_count); + unsigned int code = static_cast(compress_bits(packed, code_mask)); + uint64_t decoded = packed_quints[code]; + decoded = (decoded << bits) | deposit_bits(compress_bits(packed, low_mask), lane_mask); + std::memcpy(output_data + i, &decoded, 3); + bit_offset += bit_count; + } + + unsigned int tail_count = count - i; + if (tail_count != 0) + { + bit_count = tail_count * bits + (tail_count * 7 + 2) / 3; + uint64_t packed = read_ise_group_packed(input_data, bit_offset, bit_count); + unsigned int code = static_cast(compress_bits(packed, code_mask)); + uint64_t decoded = packed_quints[code]; + decoded = (decoded << bits) | deposit_bits(compress_bits(packed, low_mask), lane_mask); + for (unsigned int j = 0; j < tail_count; j++) + { + output_data[i + j] = static_cast(decoded >> (8 * j)); + } + } +} + /* See header for documentation. */ void encode_ise( quant_method quant_level, @@ -644,9 +961,21 @@ void encode_ise( } } -/* See header for documentation. */ -void decode_ise( - quant_method quant_level, +/** + * @brief Decode an ISE sequence with explicit binary and radix parameters. + * + * @param bits The binary bits per value. + * @param is_trit_sequence True if the sequence has trit groups. + * @param is_quint_sequence True if the sequence has quint groups. + * @param character_count The number of encoded values. + * @param input_data The packed input sequence. + * @param[out] output_data The decoded values. + * @param bit_offset The initial input bit offset. + */ +static ASTCENC_SIMD_INLINE void decode_ise_fixed( + unsigned int bits, + bool is_trit_sequence, + bool is_quint_sequence, unsigned int character_count, const uint8_t* input_data, uint8_t* output_data, @@ -654,82 +983,390 @@ void decode_ise( ) { promise(character_count > 0); - // Trit/quint unpacking can write beyond character_count. Reserve four padding entries after the - // maximum 64 results. - uint8_t results[68]; - uint8_t tq_blocks[22] { 0 }; // Trit-blocks or quint-blocks, must be zeroed + if constexpr (is_bit_permute_fast) + { + if (is_trit_sequence) + { + if (bits == 0) + { + decode_ise_trits_packed(bits, character_count, input_data, output_data, bit_offset); + } + else + { + decode_ise_trits_packed(bits, character_count, input_data, output_data, bit_offset); + } - unsigned int bits = btq_counts[quant_level].bits; - unsigned int trits = btq_counts[quant_level].trits; - unsigned int quints = btq_counts[quant_level].quints; + return; + } - unsigned int lcounter = 0; - unsigned int hcounter = 0; + if (is_quint_sequence) + { + if (bits == 0) + { + decode_ise_quints_packed(bits, character_count, input_data, output_data, bit_offset); + } + else + { + decode_ise_quints_packed(bits, character_count, input_data, output_data, bit_offset); + } - // Collect bits for each element, as well as bits for any trit-blocks and quint-blocks - for (unsigned int i = 0; i < character_count; i++) + return; + } + + decode_ise_bits_packed(bits, character_count, input_data, output_data, bit_offset); + } + else { - results[i] = static_cast(read_bits(bits, bit_offset, input_data)); - bit_offset += bits; + // Trit/quint unpacking can write beyond character_count. Reserve four padding entries after + // the maximum 64 results. + uint8_t results[68]; + uint8_t tq_blocks[22] { 0 }; // Trit-blocks or quint-blocks, must be zeroed - if (trits) + unsigned int lcounter = 0; + unsigned int hcounter = 0; + + // Collect bits for each element, as well as bits for any trit-blocks and quint-blocks + for (unsigned int i = 0; i < character_count; i++) { - static const uint8_t bits_to_read[5] { 2, 2, 1, 2, 1 }; - static const uint8_t block_shift[5] { 0, 2, 4, 5, 7 }; - static const uint8_t next_lcounter[5] { 1, 2, 3, 4, 0 }; - static const uint8_t hcounter_incr[5] { 0, 0, 0, 0, 1 }; - unsigned int tdata = read_bits(bits_to_read[lcounter], bit_offset, input_data); - bit_offset += bits_to_read[lcounter]; - tq_blocks[hcounter] |= tdata << block_shift[lcounter]; - hcounter += hcounter_incr[lcounter]; - lcounter = next_lcounter[lcounter]; + results[i] = static_cast(read_bits(bits, bit_offset, input_data)); + bit_offset += bits; + + if (is_trit_sequence) + { + static const uint8_t bits_to_read[5] { 2, 2, 1, 2, 1 }; + static const uint8_t block_shift[5] { 0, 2, 4, 5, 7 }; + static const uint8_t next_lcounter[5] { 1, 2, 3, 4, 0 }; + static const uint8_t hcounter_incr[5] { 0, 0, 0, 0, 1 }; + unsigned int tdata = read_bits(bits_to_read[lcounter], bit_offset, input_data); + bit_offset += bits_to_read[lcounter]; + tq_blocks[hcounter] |= tdata << block_shift[lcounter]; + hcounter += hcounter_incr[lcounter]; + lcounter = next_lcounter[lcounter]; + } + + if (is_quint_sequence) + { + static const uint8_t bits_to_read[3] { 3, 2, 2 }; + static const uint8_t block_shift[3] { 0, 3, 5 }; + static const uint8_t next_lcounter[3] { 1, 2, 0 }; + static const uint8_t hcounter_incr[3] { 0, 0, 1 }; + unsigned int tdata = read_bits(bits_to_read[lcounter], bit_offset, input_data); + bit_offset += bits_to_read[lcounter]; + tq_blocks[hcounter] |= tdata << block_shift[lcounter]; + hcounter += hcounter_incr[lcounter]; + lcounter = next_lcounter[lcounter]; + } } - if (quints) + // Unpack trit-blocks or quint-blocks as needed + if (is_trit_sequence) { - static const uint8_t bits_to_read[3] { 3, 2, 2 }; - static const uint8_t block_shift[3] { 0, 3, 5 }; - static const uint8_t next_lcounter[3] { 1, 2, 0 }; - static const uint8_t hcounter_incr[3] { 0, 0, 1 }; - unsigned int tdata = read_bits(bits_to_read[lcounter], bit_offset, input_data); - bit_offset += bits_to_read[lcounter]; - tq_blocks[hcounter] |= tdata << block_shift[lcounter]; - hcounter += hcounter_incr[lcounter]; - lcounter = next_lcounter[lcounter]; + unsigned int trit_blocks = (character_count + 4) / 5; + promise(trit_blocks > 0); + for (unsigned int i = 0; i < trit_blocks; i++) + { + const uint8_t *tritptr = trits_of_integer[tq_blocks[i]]; + results[5 * i ] |= tritptr[0] << bits; + results[5 * i + 1] |= tritptr[1] << bits; + results[5 * i + 2] |= tritptr[2] << bits; + results[5 * i + 3] |= tritptr[3] << bits; + results[5 * i + 4] |= tritptr[4] << bits; + } } - } - // Unpack trit-blocks or quint-blocks as needed - if (trits) - { - unsigned int trit_blocks = (character_count + 4) / 5; - promise(trit_blocks > 0); - for (unsigned int i = 0; i < trit_blocks; i++) + if (is_quint_sequence) { - const uint8_t *tritptr = trits_of_integer[tq_blocks[i]]; - results[5 * i ] |= tritptr[0] << bits; - results[5 * i + 1] |= tritptr[1] << bits; - results[5 * i + 2] |= tritptr[2] << bits; - results[5 * i + 3] |= tritptr[3] << bits; - results[5 * i + 4] |= tritptr[4] << bits; + unsigned int quint_blocks = (character_count + 2) / 3; + promise(quint_blocks > 0); + for (unsigned int i = 0; i < quint_blocks; i++) + { + const uint8_t *quintptr = quints_of_integer[tq_blocks[i]]; + results[3 * i ] |= quintptr[0] << bits; + results[3 * i + 1] |= quintptr[1] << bits; + results[3 * i + 2] |= quintptr[2] << bits; + } } - } - if (quints) - { - unsigned int quint_blocks = (character_count + 2) / 3; - promise(quint_blocks > 0); - for (unsigned int i = 0; i < quint_blocks; i++) + for (unsigned int i = 0; i < character_count; i++) { - const uint8_t *quintptr = quints_of_integer[tq_blocks[i]]; - results[3 * i ] |= quintptr[0] << bits; - results[3 * i + 1] |= quintptr[1] << bits; - results[3 * i + 2] |= quintptr[2] << bits; + output_data[i] = results[i]; } } +} - for (unsigned int i = 0; i < character_count; i++) - { - output_data[i] = results[i]; - } +/* See header for documentation. */ +void decode_ise( + quant_method quant_level, + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + const auto& counts = btq_counts[quant_level]; + decode_ise_fixed(counts.bits, counts.trits != 0, counts.quints != 0, + character_count, input_data, output_data, bit_offset); +} + +#if defined(ASTCENC_RDO) + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_2( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(1, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_3( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(0, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_4( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(2, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_5( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(0, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_6( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(1, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_8( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(3, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_10( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(1, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_12( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(2, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_16( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(4, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_20( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(2, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_24( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(3, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_32( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(5, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_40( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(3, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_48( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(4, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_64( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(6, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_80( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(4, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_96( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(5, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_128( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(7, false, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_160( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(5, false, true, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_192( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(6, true, false, character_count, + input_data, output_data, bit_offset); +} + +/** @copydoc rdo_ise_decode_func */ +static void rdo_decode_ise_quant_256( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset +) { + decode_ise_fixed(8, false, false, character_count, + input_data, output_data, bit_offset); +} + +/* See header for documentation. */ +rdo_ise_decode_func* rdo_get_ise_decoder(unsigned int quant_level) +{ + static rdo_ise_decode_func* const decoders[] { + rdo_decode_ise_quant_2, + rdo_decode_ise_quant_3, + rdo_decode_ise_quant_4, + rdo_decode_ise_quant_5, + rdo_decode_ise_quant_6, + rdo_decode_ise_quant_8, + rdo_decode_ise_quant_10, + rdo_decode_ise_quant_12, + rdo_decode_ise_quant_16, + rdo_decode_ise_quant_20, + rdo_decode_ise_quant_24, + rdo_decode_ise_quant_32, + rdo_decode_ise_quant_40, + rdo_decode_ise_quant_48, + rdo_decode_ise_quant_64, + rdo_decode_ise_quant_80, + rdo_decode_ise_quant_96, + rdo_decode_ise_quant_128, + rdo_decode_ise_quant_160, + rdo_decode_ise_quant_192, + rdo_decode_ise_quant_256, + }; + assert(quant_level < 21); + return decoders[quant_level]; } +#endif diff --git a/Source/astcenc_internal.h b/Source/astcenc_internal.h index 669b0e7ad..8dc81ed86 100644 --- a/Source/astcenc_internal.h +++ b/Source/astcenc_internal.h @@ -2165,17 +2165,17 @@ void compute_angular_endpoints_2planes( ============================================================================ */ /** - * @brief Compress an image block into a physical block. + * @brief Compress an image block into a symbolic block. * * @param ctx The compressor context and configuration. * @param blk The image block color data to compress. - * @param[out] pcb The physical compressed block output. + * @param[out] scb The symbolic compressed block output. * @param[out] tmpbuf Preallocated scratch buffers for the compressor. */ void compress_block( const astcenc_contexti& ctx, const image_block& blk, - uint8_t pcb[16], + symbolic_compressed_block& scb, compression_working_buffers& tmpbuf); /** diff --git a/Source/astcenc_internal_entry.h b/Source/astcenc_internal_entry.h index a7dca6c72..5ef66abb0 100644 --- a/Source/astcenc_internal_entry.h +++ b/Source/astcenc_internal_entry.h @@ -201,10 +201,14 @@ class ParallelManager * This can be called from multi-threaded code. The first thread to hit this will process the * initialization. Other threads will block and wait for it to complete. * - * @param task_count Total number of tasks needing processing. - * @param callback Function pointer for progress status callbacks. + * @param task_count Total number of tasks needing processing. + * @param callback Function pointer for progress status callbacks. + * @param callback_min_tasks Minimum task count between progress callbacks. */ - void init(size_t task_count, astcenc_progress_callback callback) + void init( + size_t task_count, + astcenc_progress_callback callback, + size_t callback_min_tasks = 4096) { std::lock_guard lck(m_lock); @@ -214,8 +218,9 @@ class ParallelManager m_total_task_count = task_count; m_init_done = true; - // Report every 1% or 4096 blocks, whichever is larger, to avoid callback overhead - float min_diff = (4096.0f / static_cast(task_count)) * 100.0f; + // Report every 1% or the requested task minimum to avoid callback overhead + float min_diff = (static_cast(callback_min_tasks) / + static_cast(task_count)) * 100.0f; m_callback_min_diff = astc::max(min_diff, 1.0f); } } diff --git a/Source/astcenc_morton.h b/Source/astcenc_morton.h new file mode 100644 index 000000000..e3c436d87 --- /dev/null +++ b/Source/astcenc_morton.h @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @brief Brick layouts for encoder scheduling and RDO. + * + * These dimensions are independent of the fixed 8x8 bricks used by the .astcm file format. + */ + +#ifndef ASTCENC_MORTON_INCLUDED +#define ASTCENC_MORTON_INCLUDED + +#include +#include +#include +#include +#include + +/** @brief A 3D position, in units specified by the caller. */ +struct position +{ + /** @brief The X coordinate. */ + size_t x; + + /** @brief The Y coordinate. */ + size_t y; + + /** @brief The Z coordinate. */ + size_t z; +}; + +/** + * @brief A square or 2:1 Morton brick with power-of-two dimensions. + * + * @tparam dim_x The brick width in ASTC blocks. + * @tparam dim_y The brick height in ASTC blocks. + */ +template +struct morton_brick_layout +{ + static_assert(std::has_single_bit(dim_x) && std::has_single_bit(dim_y), + "Morton brick dimensions must be non-zero powers of two"); + + static_assert((dim_x == dim_y) || ((dim_x / 2 == dim_y) && (dim_x > dim_y)), + "Morton bricks must be 1:1 or 2:1 width-to-height ratio"); + + static_assert(dim_x <= 16, + "Morton brick must be less than or equal to 16 blocks"); + + static_assert(dim_y <= 16, + "Morton brick must be less than or equal to 16 blocks"); + + /** @brief The brick width in ASTC blocks. */ + static constexpr unsigned int width = dim_x; + + /** @brief The brick height in ASTC blocks. */ + static constexpr unsigned int height = dim_y; + + /** @brief The number of Morton positions in the brick. */ + static constexpr unsigned int block_count = dim_x * dim_y; + + /** @brief A validity bit for each Morton position, including sparse edge bricks. */ + using validity_mask = std::bitset; + + /** + * @brief Decode a dense Morton index into local block coordinates. + * + * A 2:1 brick appends a high X bit after the interleaved square coordinates. + * + * @param index The Morton index in the range [0, block_count). + * + * @return The local block coordinates, with Z set to zero. + */ + static constexpr position get_position( + unsigned int index + ) { + unsigned int x = 0; + unsigned int y = 0; + for (unsigned int bit = 1; bit < height; bit <<= 1) + { + x |= (index & 1) * bit; + y |= ((index >> 1) & 1) * bit; + index >>= 2; + } + + if constexpr (width > height) + { + x |= (index & 1) * height; + } + + return { .x = x, .y = y, .z = 0 }; + } +}; + +/** @brief The selected encoder and RDO brick layout, independent of .astcm file tiles. */ +struct brick_layout +{ + /** @brief The number of block positions in either layout. */ + static constexpr unsigned int block_count = 128; + + /** @brief A validity bit for each position, including sparse edge bricks. */ + using validity_mask = std::bitset; + + /** @brief The brick width in ASTC blocks. */ + unsigned int width { 128 }; + + /** @brief The brick height in ASTC blocks. */ + unsigned int height { 1 }; + + /** + * @brief Select the brick dimensions and traversal order. + * + * @param is_morton Whether to use Morton order instead of scanline order. + */ + brick_layout(bool is_morton) + { + if (is_morton) + { + width = 16; + height = 8; + } + } + + /** + * @brief Decode a brick index into local block coordinates in the selected order. + * + * @param index The block index in the range [0, block_count). + * + * @return The local block coordinates, with Z set to zero. + */ + position get_position(unsigned int index) const + { + if (height == 1) + { + return { .x = index, .y = 0, .z = 0 }; + } + + return morton_brick_layout<16, 8>::get_position(index); + } +}; + +#endif diff --git a/Source/astcenc_rdo.h b/Source/astcenc_rdo.h new file mode 100644 index 000000000..973a6b3d7 --- /dev/null +++ b/Source/astcenc_rdo.h @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief RDO brick integration, byte search, rate estimation, and distortion interfaces. + * + * Rate-distortion optimization (RDO) trades image quality for smaller distribution files. It + * improves the compression ratio that can be achieved when applying lossless compression, such as + * ZIP or LZ4, to an ASTC compressed image. + * + * RDO is a functionally transparent optimization; the output of the RDO process is still a normal + * compressed ASTC block that can be used directly by the GPU. The data has just been conditioned + * so that bit patterns are more similar to surrounding blocks, making it more compressible. + * + * Entropy reduction transform + * =========================== + * + * The algorithm used by ASTCEnc is based on Rich Geldreich's public-domain Entropy Reduction + * Transform (ERT) implementation in bc7enc_rdo. + * + * See https://github.com/richgel999/bc7enc_rdo/blob/master/ert.cpp. + * + * The original ERT implementation is a generic implementation that is oblivious to the data format + * that is it transforming. This makes it trivial to integrate with any data source, as it only + * needs some cost function to use, but it makes applying the ERT very slow. Compressing the Kodak + * suite using ASTC -thorough compression takes around 1.6 seconds. Applying the out-of-the-box ERT + * transform with a 1KB history buffer takes 60 seconds at lambda=0.2. This is really too slow to + * apply at scale on a large project. + * + * This implementation is a heavily optimized version of the original ERT transform. It includes + * both generic optimizations, and format-specific optimizations that exploit the fact that we know + * it is transforming compressed ASTC blocks. On our benchmarking machine the specialized ASTC ERT + * (AERT) implementation takes just 3.5 seconds for similar PSNR and ZIP-compressed file size. + * + * Algorithm + * ========= + * + * When using RDO the main ASTC compression will generate a "brick" containing 2KB of compressed + * data (128 compressed ASTC blocks). Bricks are either arranged linearly, or as a 16x8 Morton + * order block. Each encoder brick is AERT optimized independently. + * + * The AERT process is: + * + * 1. Gather valid physical blocks into a compact stream in brick traversal order. + * 2. Identify protected bits that cannot be mutated by the ERT. We allow only color endpoints and + * weights to change, protecting everything else. + * 2. Learn DEFLATE literal, length, and distance Huffman prices from a greedy parse of the brick. + * A sliding history follows the accepted prefix; token prices are refreshed at the midpoint. + * 3. Visit blocks in stream order. Try copying byte ranges from the same byte offsets in nearby + * predecessors, then optionally add a second non-overlapping range to the best first match. + * Structural masks, candidate hashes, and search budgets prune trials before decoding. + * 4. Minimize weighted MSE * smoothness_scale + internal_lambda * estimated_bits. MSE uses + * normalized channel weights in the 0..255 domain; smooth blocks receive extra protection. + * Decode only affected payloads and stop scoring once the trial cannot beat the best cost. + * Internal lambda is the user lambda multiplied by rdo_lambda_scale; zero disables RDO. + * 5. Keep each winning block's exact physical bytes as history for subsequent blocks, then + * scatter the optimized brick back to the output stream. . + * + * The rate model estimates storage cost without running a lossless compressor for every trial. + * It caches optimal parses of unchanged byte intervals and adds the proposed match costs. + * Matches can continue across ASTC block boundaries, but history resets at brick boundaries. + * Estimated improvements need not reduce the final DEFLATE file size by the same amount. + * + * Limitaitons + * =========== + * + * In the current implementation, only LDR textures are supported. + */ + +#ifndef ASTCENC_RDO_INCLUDED +#define ASTCENC_RDO_INCLUDED + +#include +#include +#include +#include + +#include "astcenc_morton.h" + +struct astcenc_config; +struct astcenc_contexti; +struct astcenc_image; +struct astcenc_swizzle; +struct block_size_descriptor; +struct image_block; +struct rdo_brick_context; +struct symbolic_compressed_block; + +/* ============================================================================ + Compile-time tuning + + Lambda and brick traversal order are runtime controls in astcenc_config. + Brick layouts are defined by brick_layout in astcenc_morton.h. +============================================================================ */ + +/** @brief Scale user lambda to roughly match the original ERT quality and size tradeoff. */ +static constexpr float rdo_lambda_scale { 3.0f }; + +/** @brief Sliding rate-model history in bytes; a positive multiple of 16. */ +static constexpr unsigned int rdo_history_bytes { 1024 }; + +/** @brief Minimum predecessor scan budget for large ASTC footprints. */ +static constexpr unsigned int rdo_min_lookback { 32 }; + +/** @brief Maximum predecessor scan budget for small ASTC footprints. */ +static constexpr unsigned int rdo_max_lookback { 64 }; + +/** @brief Base scan budget before the inverse block-dimension adjustment; an even count. */ +static constexpr unsigned int rdo_search_base_lookback { 22 }; + +/** @brief Inverse-dimension scale in predecessor pairs; larger values search more history. */ +static constexpr unsigned int rdo_search_dimension_scale { 30 }; + +/** @brief Minimum estimated saving over the first match required for a second match, in bits. */ +static constexpr float rdo_min_second_match_saving { 16.0f }; + +/** @brief Maximum physical bit changes allowed in a nonconstant trial, in [0, 128]. */ +static constexpr unsigned int rdo_max_changed_bits { 32 }; + +/** @brief Trial RMS ceiling relative to the original, with an original MSE floor of one. */ +static constexpr float rdo_max_rms_increase { 10.0f }; + +/** @brief Source standard deviation below which smooth-block protection applies. */ +static constexpr float rdo_max_smooth_std_dev { 18.0f }; + +/** @brief Maximum distortion multiplier for smooth blocks. */ +static constexpr float rdo_smooth_mse_scale { 10.0f }; + +/** @brief True to search for a second non-overlapping copied byte range. */ +static constexpr bool rdo_is_two_matches_enabled { true }; + +/** @brief True to refresh Huffman prices after optimizing half the brick. */ +static constexpr bool rdo_is_midpoint_refresh_enabled { true }; + +/** @brief Candidate deduplication entries; a power of two trading cache size against collisions. */ +static constexpr unsigned int rdo_trial_hash_entries { 8192 }; + +/** @brief Rate-dictionary index bits, in [1, 16]; more bits shorten collision chains. */ +static constexpr unsigned int rdo_rate_hash_bits { 12 }; + +static_assert(rdo_max_smooth_std_dev > 0.0f); +static_assert(rdo_smooth_mse_scale > 0.0f); +static_assert((rdo_history_bytes > 0) && ((rdo_history_bytes % 16) == 0)); +static_assert((rdo_min_lookback > 0) && (rdo_min_lookback <= rdo_max_lookback)); +static_assert(((rdo_min_lookback | rdo_max_lookback) & 1u) == 0); +static_assert((rdo_search_base_lookback & 1u) == 0); +static_assert(rdo_max_changed_bits <= 128); +static_assert(std::has_single_bit(rdo_trial_hash_entries)); +static_assert((rdo_rate_hash_bits >= 1) && (rdo_rate_hash_bits <= 16)); + +/* ============================================================================ + Type aliases +============================================================================ */ + +/** + * @brief Decode a bounded integer sequence using a fixed quantization alphabet. + * + * @param character_count The number of encoded values. + * @param[in] input_data The packed input sequence. + * @param[out] output_data The decoded values. + * @param bit_offset The initial bit offset in the input sequence. + */ +using rdo_ise_decode_func = void( + unsigned int character_count, + const uint8_t* input_data, + uint8_t* output_data, + unsigned int bit_offset); + +/* ============================================================================ + Structures +============================================================================ */ + +/** @brief Parameters for the entropy reduction transform. */ +struct rdo_parameters +{ + /** @brief Rate weight in the weighted MSE plus estimated bits objective. */ + float lambda { 0.0f }; + + /** @brief Maximum number of predecessors searched for speculative copied ranges. */ + unsigned int search_lookback { rdo_max_lookback }; +}; + +/** @brief Original block distortion and source statistics used to initialize the search. */ +struct rdo_block_statistics +{ + /** @brief The original block's weighted MSE in the 0..255 domain. */ + float mse; + + /** @brief The source standard deviation used for smooth-block protection. */ + float std_dev; +}; + +/** @brief A byte match selected for the current block. */ +struct rdo_byte_match +{ + /** @brief The match length in bytes; zero indicates no match. */ + unsigned int length { 0 }; + + /** @brief The absolute source byte offset in the input stream. */ + unsigned int source { 0 }; + + /** @brief The destination byte offset within the current block. */ + unsigned int offset { 0 }; + + /** @brief The estimated cost of the match in bits. */ + float bits { 0.0f }; +}; + +/** @brief Fixed ISE layout for trials retaining the original nonconstant block structure. */ +struct rdo_block_decode_state +{ + /** @brief Fixed-quantization ISE decoder for weight indices. */ + rdo_ise_decode_func* weight_decode; + + /** @brief Fixed-quantization ISE decoder for endpoint indices. */ + rdo_ise_decode_func* color_decode; + + /** @brief The number of weights in each plane. */ + unsigned int weight_count; + + /** @brief The total number of encoded weights. */ + unsigned int real_weight_count; + + /** @brief The weight quantization alphabet index. */ + unsigned int weight_quant_mode; + + /** @brief The total number of endpoint integers. */ + unsigned int color_count; + + /** @brief The endpoint sequence's bit offset. */ + unsigned int color_offset; + + /** @brief The number of endpoint integers in each partition. */ + unsigned int color_counts[4]; + + /** @brief Bytes containing weight ISE bits. */ + unsigned int weight_byte_mask; + + /** @brief Bytes containing endpoint ISE bits. */ + unsigned int color_byte_mask; + + /** @brief True if the block uses two weight planes. */ + bool is_dual_plane; +}; + +/** + * @brief Original reconstructed values reusable when a trial leaves a payload unchanged. + * + * The caller updates the unchanged flags for each trial using conservative payload byte masks. + */ +struct rdo_block_difference_cache +{ + /** @brief Unpacked endpoint pairs for each partition. */ + int endpoints[4][2][4]; + + /** @brief Interpolated weight planes, padded for the largest supported SIMD width. */ + uint8_t weights[2][224]; + + /** @brief Original weighted squared errors in raster texel order. */ + float texel_errors[224]; + + /** @brief Maximum contributing weight count for each fixed-layout SIMD batch. */ + uint8_t weight_counts[224 / 4]; + + /** @brief True if this trial preserves the original endpoint payload. */ + bool is_colors_unchanged; + + /** @brief True if this trial preserves the original weight payload. */ + bool is_weights_unchanged; +}; + +/* ============================================================================ + Classes +============================================================================ */ + +/** @brief Brick-local Huffman prices and cached rates for unchanged byte intervals. */ +class rdo_rate_model +{ +public: + /** + * @brief Learn token prices from a greedy parse of the original brick. + * + * Prices remain fixed until the next init() call. The search retrains them halfway through + * the brick; the match dictionary follows accepted changes between retraining points. + * + * @param[in] data The original compact physical block stream. + * @param block_count The number of valid blocks. + * @param lookback The maximum history length in blocks. + */ + void init( + const uint8_t* data, + unsigned int block_count, + unsigned int lookback); + + /** + * @brief Prepare rates using the current optimized prefix. + * + * Blocks must be prepared in increasing index order between init() calls. The stream prefix + * before the current block must contain all accepted changes and remain unchanged afterward. + * + * @param[in] data The compact physical block stream. + * @param block_index The block to optimize. + * @param history_start The first retained history block. + */ + void prepare( + const uint8_t* data, + unsigned int block_index, + unsigned int history_start); + + /** + * @brief Estimate a match's token cost. + * + * @param distance The backward distance in bytes. + * @param length The match length in bytes, in the range 3..16. + * + * @return The match cost in bits. + */ + float get_match_cost( + unsigned int distance, + unsigned int length) const; + + /** + * @brief Estimate a block using zero, one, or two copied ranges. + * + * Bytes outside the copied ranges must match the original block passed to prepare(). + * The two ranges must not overlap. A zero-length first match requests the original cost. + * Estimates at or above that original cost cannot improve the search and must be rejected. + * + * @param[in] first The first match, or an empty match for the original block. + * @param[in] second The second match, or an empty match for a single-match trial. + * + * @return The estimated block cost in bits. + */ + float estimate( + const rdo_byte_match& first, + const rdo_byte_match& second) const; + + /** + * @brief Get the frozen Huffman price of a literal byte. + * + * @param value The literal byte. + * + * @return The literal cost in bits. + */ + float get_literal_cost( + uint8_t value) const; + +private: + static_assert(brick_layout::block_count <= 32768 / 16, + "RDO brick distances must fit in the DEFLATE window"); + + /** @brief The number of input bytes in a full encoder brick. */ + static constexpr unsigned int max_bytes = brick_layout::block_count * 16; + + /** @brief The number of buckets in the three-byte match dictionary. */ + static constexpr unsigned int hash_size = 1u << rdo_rate_hash_bits; + + /** @brief Frozen code lengths for literal, length, and end symbols. */ + uint8_t literal_length_bits[286]; + + /** @brief Length-symbol and extra-bit costs, indexed by match length. */ + uint8_t length_costs[17]; + + /** @brief Distance-symbol and extra-bit costs, indexed by byte distance. */ + uint8_t distance_costs[max_bytes + 1]; + + /** @brief Newest completed source substring in each hash bucket. */ + int heads[hash_size]; + + /** @brief Earlier source substring in the same hash bucket. */ + int previous[max_bytes]; + + /** @brief The first source byte not yet entered into the match dictionary. */ + unsigned int indexed_bytes; + + /** @brief Maximum match distance in the sliding history, in bytes. */ + unsigned int history_bytes; + + /** @brief The current block's unchanged physical bytes. */ + uint8_t original[16]; + + /** @brief Minimum token cost for each unchanged half-open byte interval. */ + float intervals[17][17]; +}; + +/* ============================================================================ + Brick integration +============================================================================ */ + +/** + * @brief Optimize a brick in its selected traversal order, then scatter accepted physical bytes. + * + * Disabled RDO and unsupported configurations leave the stream unchanged. + * + * @param[in] ctx The compressor context. + * @param[in] image The source image. + * @param[in] swizzle The compression swizzle. + * @param[in] valid_mask The bitmask of valid brick positions in the brick. + * @param[in] offsets The raster byte offset at each valid brick position. + * @param[in] blocks The original symbolic blocks at each valid brick position. + * @param[in,out] buffer The raster-ordered physical output stream. + */ +void rdo_optimize_physical_brick( + const astcenc_contexti& ctx, + const astcenc_image& image, + const astcenc_swizzle& swizzle, + const brick_layout::validity_mask& valid_mask, + const size_t* offsets, + const symbolic_compressed_block* blocks, + uint8_t* buffer); + +/** + * @brief Load an original block and initialize its distortion caches and source statistics. + * + * @param[in,out] ctx The brick distortion context. + * @param block_index The compact block index in brick traversal order. + * + * @return The original weighted MSE and source standard deviation. + */ +rdo_block_statistics rdo_initialize_block( + rdo_brick_context& ctx, + unsigned int block_index); + +/** + * @brief Evaluate a physical trial using decoded weighted MSE. + * + * Initialize the current block with rdo_initialize_block() before scoring trials. byte_mask may + * include unchanged bytes. + * + * @param[in,out] ctx The initialized brick distortion context. + * @param[in] physical The trial physical block. + * @param byte_mask A conservative mask of changed physical bytes. + * @param mse_limit The MSE bound, or infinity for a complete score. + * + * @return Weighted MSE in the 0..255 domain, or @c std::numeric_limits::max() + * for losing trials. + */ +float rdo_block_difference( + rdo_brick_context& ctx, + const uint8_t* physical, + unsigned int byte_mask, + float mse_limit); + +/* ============================================================================ + Search and rate estimation +============================================================================ */ + +/** + * @brief Prepare polynomial prefixes for all substrings of one physical block. + * + * @param[in] data The source block bytes. + * @param[out] prefixes The seventeen polynomial prefixes, including the empty prefix. + */ +void rdo_prepare_substring_hashes( + const uint8_t data[16], + uint32_t prefixes[17]); + +/** + * @brief Compute salted hashes for every fixed-offset substring of one length. + * + * @param[in] prefixes The prefixes, padded to 16 + ASTCENC_SIMD_WIDTH initialized entries. + * @param length The substring length, in the range 3..16. + * @param[out] hashes The resulting 17 - length hashes; remaining entries are untouched. + */ +void rdo_hash_substrings( + const uint32_t* prefixes, + unsigned int length, + uint32_t hashes[16]); + +/** + * @brief Improve estimated compressibility by copying byte ranges from preceding blocks. + * + * Input blocks must be valid encoder output. History and match continuation state are local to + * this call. Constant blocks remain unchanged. Only trials preserving the protected layout bits + * reach distortion scoring. + * + * @param[in,out] ctx The brick distortion context. + * @param[in,out] data The contiguous stream of 16-byte compressed blocks. + * @param block_count The number of blocks, at most brick_layout::block_count. + * @param[in] masks The protected layout bits for each 16-byte block. + * @param[in] params The distortion and history settings. + */ +void rdo_reduce_entropy( + rdo_brick_context& ctx, + uint8_t* data, + unsigned int block_count, + const uint8_t* masks, + const rdo_parameters& params); + +/* ============================================================================ + Physical layout and payload decoding +============================================================================ */ + +/** + * @brief Select a dedicated decoder for a fixed quantization alphabet. + * + * @param quant_level The quantization alphabet index. + * + * @return The fixed-quantization ISE decoder. + */ +rdo_ise_decode_func* rdo_get_ise_decoder(unsigned int quant_level); + +/** + * @brief Decode pure trit or quint weight sequences without an intermediate index array. + * + * @param quant_level QUANT_3 or QUANT_5. + * @param count The number of stored weights. + * @param[in] input_data The reversed physical block, with the sequence at bit zero. + * @param[out] output_data The unquantized weights, with exactly count bytes written. + */ +void rdo_decode_ise_radix_weights( + unsigned int quant_level, + unsigned int count, + const uint8_t* input_data, + uint8_t* output_data); + +/** + * @brief Prepare payload decoding from the encoder's original symbolic layout. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The original valid nonconstant symbolic block. + * + * @return The fixed payload layout. + */ +rdo_block_decode_state rdo_prepare_block_decode( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block); + +/** + * @brief Decode changed payload bytes, leaving untouched payload storage unspecified. + * + * Layout bits must match the original block. The byte mask may overestimate changes, but must + * include every changed byte to avoid reusing a stale payload. + * + * @param[in] state The fixed nonconstant layout. + * @param[in] physical The candidate physical bytes. + * @param byte_mask A conservative mask of changed physical bytes. + * @param[in,out] block The trial symbolic block with its fixed layout initialized. + */ +void rdo_decode_changed_payload( + const rdo_block_decode_state& state, + const uint8_t physical[16], + unsigned int byte_mask, + symbolic_compressed_block& block); + +/** + * @brief Identify bits selecting the block layout rather than endpoint or weight values. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The original valid symbolic encoding. + * @param[out] mask The protected layout bits, or all bits for constant blocks. + */ +void rdo_get_structure_mask( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + uint8_t mask[16]); + +/* ============================================================================ + Distortion scoring +============================================================================ */ + +/** + * @brief Bound the MSE of a candidate that can improve the current rate-distortion cost. + * + * @param best_cost The current best cost. + * @param rate_cost The candidate's weighted rate cost. + * @param mse_recip The reciprocal of the positive distortion multiplier. + * + * @return The MSE limit, or zero if the rate alone cannot improve the cost. + */ +float rdo_get_mse_limit( + float best_cost, + float rate_cost, + float mse_recip); + +/** + * @brief Scale cached source texels into the distortion metric's 0..255 domain. + * + * @param[in,out] original The loaded source block, initially in the 0..65535 domain. + * @param texel_count The number of source texels. + */ +void rdo_prepare_difference_source( + image_block& original, + unsigned int texel_count); + +/** + * @brief Cache reconstructed endpoints and interpolated weights for an original block. + * + * @param[in] config The LDR compressor configuration. + * @param[in] bsd The block size descriptor. + * @param[in] block The valid nonconstant symbolic encoding from the core compressor. + * @param[out] cache The reconstructed values to reuse. + */ +void rdo_prepare_difference_cache( + const astcenc_config& config, + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + rdo_block_difference_cache& cache); + +/** + * @brief Score the original reconstruction and cache its errors without decoding it again. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The original valid nonconstant symbolic block. + * @param[in] original The prepared source texels and normalized channel weights. + * @param[in,out] cache The prepared endpoints and weights, receiving original errors. + * + * @return The original weighted MSE and source standard deviation. + */ +rdo_block_statistics rdo_compute_original_difference( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + rdo_block_difference_cache& cache); + +/** + * @brief Decode and score a full LDR block, stopping when a trial cannot win. + * + * The block must be nonconstant. The source uses normalized channel weights. + * + * @param[in] config The LDR compressor configuration. + * @param[in] bsd The block size descriptor. + * @param[in] block The symbolic encoding. + * @param[in] original The prepared 0..255 source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache, including per-texel errors. + * + * @return Weighted MSE in the 0..255 domain, or @c std::numeric_limits::max() + * for losing trials. + */ +float rdo_compute_block_difference( + const astcenc_config& config, + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache); + +#endif diff --git a/Source/astcenc_rdo_brick.cpp b/Source/astcenc_rdo_brick.cpp new file mode 100644 index 000000000..b50f10980 --- /dev/null +++ b/Source/astcenc_rdo_brick.cpp @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief Brick-local RDO integration for LDR image compression. + */ + +#if !defined(ASTCENC_DECOMPRESS_ONLY) + +#include +#include + +#include "astcenc_internal.h" +#include "astcenc_rdo.h" + +/** @brief Source data and scratch storage for one brick's distortion scoring. */ +struct rdo_brick_context +{ + /** @brief The compressor context. */ + const astcenc_contexti& ctx; + + /** @brief The original image. */ + const astcenc_image& image; + + /** @brief The compression swizzle. */ + const astcenc_swizzle& swizzle; + + /** @brief Raster byte offsets indexed by compact brick position. */ + const size_t* offsets; + + /** @brief Original symbolic blocks indexed by compact brick position. */ + const symbolic_compressed_block* const* blocks; + + /** @brief The number of blocks in an image row. */ + size_t blocks_x; + + /** @brief Cached source texels in the 0..255 distortion domain, loaded once per block. */ + image_block original; + + /** @brief Reusable trial block retaining the original layout. */ + symbolic_compressed_block symbolic; + + /** @brief ISE layout prepared once for the current nonconstant block. */ + rdo_block_decode_state decode_state; + + /** @brief Original endpoint and interpolation results reused by unchanged payloads. */ + rdo_block_difference_cache difference_cache; +}; + +/** + * @brief Test whether RDO supports and enables this configuration. + * + * @param[in] config The compressor configuration. + * + * @return True if RDO is enabled for this configuration. + */ +static bool rdo_is_enabled( + const astcenc_config& config +) { + bool is_ldr = (config.profile == ASTCENC_PRF_LDR) || + (config.profile == ASTCENC_PRF_LDR_SRGB); + + bool is_compression_enabled = !(config.flags & ASTCENC_FLG_DECOMPRESS_ONLY); + + return (config.rdo_lambda > 0.0f) + && is_ldr && (config.block_z == 1) + && is_compression_enabled; +} + +/** + * @brief Scale speculative search to the block's effective linear dimensions. + * + * This empirical fit targets about 0.3% combined DEFLATE growth on KodakMnt RGB at lambda 0.2. + * Rate estimates still retain the full sliding history. + * + * @param block_x The ASTC block width in texels. + * @param block_y The ASTC block height in texels. + * + * @return An even predecessor count within the configured search bounds. + */ +static unsigned int rdo_get_search_lookback( + unsigned int block_x, + unsigned int block_y +) { + unsigned int texel_count = block_x * block_y; + + unsigned int perimeter_budget = rdo_search_dimension_scale * (block_x + block_y); + + unsigned int search_size = rdo_search_base_lookback + + 2 * ((perimeter_budget + texel_count - 1) / texel_count); + + return astc::clamp(search_size, rdo_min_lookback, rdo_max_lookback); +} + +/** + * @brief Load a block's source data and normalized distortion weights. + * + * @param[in,out] ctx The brick distortion context. + * @param block_index The compact block index in brick traversal order. + */ +static void rdo_load_source( + rdo_brick_context& ctx, + unsigned int block_index +) { + const auto& config = ctx.ctx.config; + const auto& bsd = *ctx.ctx.bsd; + size_t raster_index = ctx.offsets[block_index] / 16; + + const position origin { + .x = (raster_index % ctx.blocks_x) * bsd.dim_x, + .y = (raster_index / ctx.blocks_x) * bsd.dim_y, + .z = 0 + }; + + // Normalize source data once; every trial for this block uses the same distortion domain + image_block& original = ctx.original; + original.texel_count = bsd.texel_count; + + load_image_block(config.profile, ctx.image, original, bsd, + origin.x, origin.y, origin.z, ctx.swizzle); + + vfloat4 weights(config.cw_r_weight, config.cw_g_weight, + config.cw_b_weight, config.cw_a_weight); + + if (config.flags & ASTCENC_FLG_USE_ALPHA_WEIGHT) + { + float alpha_scale = original.data_max.lane<3>() * (1.0f / 65535.0f); + weights = weights * vfloat4(alpha_scale, alpha_scale, alpha_scale, 1.0f); + } + + original.channel_weight = weights / hadd_s(weights); + rdo_prepare_difference_source(original, bsd.texel_count); + + // Protected layout bits let trials reuse the encoder's mode and ISE sequence boundaries + const auto& source = *ctx.blocks[block_index]; + assert(source.block_type == SYM_BTYPE_NONCONST); + ctx.symbolic = source; + ctx.decode_state = rdo_prepare_block_decode(bsd, source); + + // Initialize only the padding needed by full SIMD reads of the original weight planes + size_t weight_count = ctx.decode_state.weight_count; + size_t padding = round_up_to_simd_multiple_vla(weight_count) - weight_count; + if (padding != 0) + { + std::memset(ctx.symbolic.weights + weight_count, 0, padding); + if (ctx.decode_state.is_dual_plane) + { + std::memset(ctx.symbolic.weights + WEIGHTS_PLANE2_OFFSET + weight_count, 0, padding); + } + } +} + +/* See header for documentation. */ +rdo_block_statistics rdo_initialize_block( + rdo_brick_context& ctx, + unsigned int block_index +) { + rdo_load_source(ctx, block_index); + const auto& bsd = *ctx.ctx.bsd; + rdo_prepare_difference_cache( + ctx.ctx.config, bsd, ctx.symbolic, ctx.difference_cache); + + return rdo_compute_original_difference(bsd, ctx.symbolic, ctx.original, + ctx.difference_cache); +} + +/* See header for documentation. */ +float rdo_block_difference( + rdo_brick_context& ctx, + const uint8_t* physical, + unsigned int byte_mask, + float mse_limit +) { + const auto& bsd = *ctx.ctx.bsd; + auto& symbolic = ctx.symbolic; + rdo_decode_changed_payload(ctx.decode_state, physical, byte_mask, symbolic); + + ctx.difference_cache.is_weights_unchanged = !(byte_mask & ctx.decode_state.weight_byte_mask); + ctx.difference_cache.is_colors_unchanged = !(byte_mask & ctx.decode_state.color_byte_mask); + + return rdo_compute_block_difference(ctx.ctx.config, bsd, symbolic, ctx.original, + mse_limit, ctx.difference_cache); +} + +/* See header for documentation. */ +void rdo_optimize_physical_brick( + const astcenc_contexti& ctx, + const astcenc_image& image, + const astcenc_swizzle& swizzle, + const brick_layout::validity_mask& valid_mask, + const size_t* offsets, + const symbolic_compressed_block* blocks, + uint8_t* buffer +) { + if (!rdo_is_enabled(ctx.config) || (image.dim_z != 1)) + { + return; + } + + // Compact valid blocks in the encoder brick's traversal order, without padding + uint8_t physical[brick_layout::block_count * 16]; + uint8_t masks[brick_layout::block_count * 16]; + size_t compact_offsets[brick_layout::block_count]; + const symbolic_compressed_block* compact_blocks[brick_layout::block_count]; + unsigned int block_count = 0; + + for (unsigned int index = 0; index < brick_layout::block_count; index++) + { + if (!valid_mask[index]) + { + continue; + } + + compact_offsets[block_count] = offsets[index]; + compact_blocks[block_count] = blocks + index; + std::memcpy(physical + block_count * 16, buffer + offsets[index], 16); + rdo_get_structure_mask(*ctx.bsd, blocks[index], masks + block_count * 16); + block_count++; + } + + if (block_count < 2) + { + return; + } + + rdo_brick_context context { + .ctx = ctx, + .image = image, + .swizzle = swizzle, + .offsets = compact_offsets, + .blocks = compact_blocks, + .blocks_x = astc::get_block_count_safe(image.dim_x, ctx.bsd->dim_x), + .original = {}, + .symbolic = {}, + .decode_state = {}, + .difference_cache = {} + }; + + const rdo_parameters params { + .lambda = ctx.config.rdo_lambda * rdo_lambda_scale, + .search_lookback = rdo_get_search_lookback(ctx.bsd->dim_x, ctx.bsd->dim_y) + }; + + rdo_reduce_entropy(context, physical, block_count, masks, params); + + // Repacking through symbolic form could discard byte matches in ignored physical bits + for (unsigned int i = 0; i < block_count; i++) + { + std::memcpy(buffer + compact_offsets[i], physical + i * 16, 16); + } +} + +#endif diff --git a/Source/astcenc_rdo_decode.cpp b/Source/astcenc_rdo_decode.cpp new file mode 100644 index 000000000..5149a13db --- /dev/null +++ b/Source/astcenc_rdo_decode.cpp @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief RDO payload decoding using the encoder's original symbolic layout. + */ + +#if !defined(ASTCENC_DECOMPRESS_ONLY) + +#include +#include + +#include "astcenc_internal.h" +#include "astcenc_rdo.h" + +/** + * @brief Mark a contiguous range of physical bits. + * + * @param[in,out] mask The physical bit mask. + * @param first The first bit to mark. + * @param count The number of bits to mark. + */ +static void rdo_mark_bits( + uint8_t mask[16], + unsigned int first, + unsigned int count +) { + for (unsigned int bit = first; bit < first + count; bit++) + { + mask[bit / 8] |= static_cast(1u << (bit % 8)); + } +} + +/* See header for documentation. */ +void rdo_get_structure_mask( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + uint8_t mask[16] +) { + std::fill_n(mask, 16, 0); + if (block.block_type != SYM_BTYPE_NONCONST) + { + // Constant blocks are retained unchanged in the byte history + std::fill_n(mask, 16, 0xFF); + return; + } + + // Layout fields surround both payloads; protecting only the low header would miss CEM bits + unsigned int header_bits = 17; + if (block.partition_count > 1) + { + header_bits = 19 + PARTITION_INDEX_BITS; + } + + rdo_mark_bits(mask, 0, header_bits); + const auto& mode = bsd.get_block_mode(block.block_mode); + unsigned int below_weights = 128 - mode.weight_bits; + if ((block.partition_count > 1) && !block.color_formats_matched) + { + unsigned int format_bits = 3 * block.partition_count - 4; + below_weights -= format_bits; + rdo_mark_bits(mask, below_weights, format_bits); + } + + if (mode.is_dual_plane) + { + rdo_mark_bits(mask, below_weights - 2, 2); + } +} + +/* See header for documentation. */ +rdo_block_decode_state rdo_prepare_block_decode( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block +) { + assert(block.block_type == SYM_BTYPE_NONCONST); + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + + rdo_block_decode_state state { + .weight_decode = rdo_get_ise_decoder(mode.quant_mode), + .color_decode = rdo_get_ise_decoder(block.quant_mode), + .weight_count = decimation.weight_count, + .real_weight_count = decimation.weight_count, + .weight_quant_mode = mode.quant_mode, + .color_count = 0, + .color_offset = 17, + .color_counts = {}, + .weight_byte_mask = 0, + .color_byte_mask = 0, + .is_dual_plane = mode.is_dual_plane != 0 + }; + + if (state.is_dual_plane) + { + state.real_weight_count *= 2; + } + + if (block.partition_count > 1) + { + state.color_offset = 19 + PARTITION_INDEX_BITS; + } + + for (unsigned int i = 0; i < block.partition_count; i++) + { + state.color_counts[i] = 2 * (block.color_formats[i] >> 2) + 2; + state.color_count += state.color_counts[i]; + } + + // Byte masks may overlap at shared payload boundaries, conservatively decoding both streams + unsigned int first_weight_byte = (128 - mode.weight_bits) / 8; + state.weight_byte_mask = 0xFFFFu & (~0u << first_weight_byte); + + unsigned int color_end = state.color_offset + + get_ise_sequence_bitcount(state.color_count, block.quant_mode); + + state.color_byte_mask = ((1u << ((color_end + 7) / 8)) - 1) + & (~0u << (state.color_offset / 8)); + return state; +} + +/** + * @brief Decode and unquantize the fixed-layout weight sequence. + * + * @param[in] state The fixed nonconstant layout. + * @param[in] physical The physical block bytes. + * @param[in,out] block The symbolic block receiving the decoded values. + */ +static void rdo_decode_weight_payload( + const rdo_block_decode_state& state, + const uint8_t physical[16], + symbolic_compressed_block& block +) { + // Weight ISE runs backward from bit 127; layout bits beyond the sequence are ignored + uint8_t reversed[16]; + for (unsigned int i = 0; i < 16; i++) + { + unsigned int value = physical[15 - i]; + value = ((value & 0x0F) << 4) | ((value >> 4) & 0x0F); + value = ((value & 0x33) << 2) | ((value >> 2) & 0x33); + value = ((value & 0x55) << 1) | ((value >> 1) & 0x55); + reversed[i] = static_cast(value); + } + + if (!state.is_dual_plane + && ((state.weight_quant_mode == QUANT_3) || (state.weight_quant_mode == QUANT_5))) + { + // Pure trit and quint sequences can combine ISE decoding with weight unquantization + rdo_decode_ise_radix_weights(state.weight_quant_mode, state.weight_count, reversed, block.weights); + return; + } + + // Binary quantizers can expand directly into mapped weight bytes + static constexpr uint8_t binary_bits[12] { 1, 0, 2, 0, 0, 3, 0, 0, 4, 0, 0, 5 }; + unsigned int bits = binary_bits[state.weight_quant_mode]; + if (!state.is_dual_plane && (bits != 0)) + { + const auto& transfer = quant_and_xfer_tables[state.weight_quant_mode]; + vtable4_16x8 low_table; + vtable_prepare(low_table, transfer.unscramble_and_unquant_map); + vtable4_32x8 table; + vtable_prepare(table, transfer.unscramble_and_unquant_map); + + for (unsigned int i = 0; i < state.weight_count; i += 8) + { + unsigned int bit_offset = i * bits; + unsigned int byte_offset = std::min(bit_offset >> 3, 8u); + + uint64_t packed = load_le64(reversed + byte_offset); + packed >>= bit_offset - byte_offset * 8; + + vint4 indices = unpack_byte_fields(packed, bits); + + vint4 weights; + if (bits == 5) + { + weights = vtable_lookup_8bit(table, indices); + } + else + { + weights = vtable_lookup_8bit(low_table, indices); + } + + // The rounded batch remains inside the 64 stored weight slots + store_low_8bytes(weights, block.weights + i); + } + + return; + } + + // Mixed-radix and dual-plane layouts share the general ISE decoder + uint8_t indices[80]; + auto weight_quant = static_cast(state.weight_quant_mode); + state.weight_decode(state.real_weight_count, reversed, indices, 0); + const auto& transfer = quant_and_xfer_tables[weight_quant]; + if (state.is_dual_plane) + { + // Padding keeps the final eight-pair batch inside allocated, unused weight slots + static_assert(WEIGHTS_PLANE2_OFFSET == 32); + std::memset(indices + state.real_weight_count, 0, 16); + vtable4_32x8 table; + vtable_prepare(table, transfer.unscramble_and_unquant_map); + for (unsigned int i = 0; i < state.weight_count; i += 8) + { + vint4 values = vint4::load(indices + 2 * i); + vint4 weights = vtable_lookup_8bit(table, values); + store_deinterleaved_bytes(weights, block.weights + i, block.weights + WEIGHTS_PLANE2_OFFSET + i); + } + } + else + { + vtable_32x8 unquant_table; + vtable_prepare(unquant_table, transfer.unscramble_and_unquant_map); + unsigned int i = 0; + for (; i + ASTCENC_SIMD_WIDTH <= state.weight_count; i += ASTCENC_SIMD_WIDTH) + { + vint weights = vtable_lookup_32bit(unquant_table, vint(indices + i)); + pack_and_store_low_bytes(weights, block.weights + i); + } + + for (; i < state.weight_count; i++) + { + block.weights[i] = transfer.unscramble_and_unquant_map[indices[i]]; + } + } +} + +/** + * @brief Decode and unquantize the fixed-layout endpoint sequence. + * + * @param[in] state The fixed nonconstant layout. + * @param[in] physical The physical block bytes. + * @param[in,out] block The symbolic block receiving the decoded values. + */ +static void rdo_decode_color_payload( + const rdo_block_decode_state& state, + const uint8_t physical[16], + symbolic_compressed_block& block +) { + uint8_t values[32]; + state.color_decode(state.color_count, physical, values, state.color_offset); + const auto* unpack = color_scrambled_pquant_to_uquant_tables[block.quant_mode - QUANT_6]; + unsigned int offset = 0; + for (unsigned int i = 0; i < block.partition_count; i++) + { + for (unsigned int j = 0; j < state.color_counts[i]; j++) + { + block.color_values[i][j] = unpack[values[offset + j]]; + } + + offset += state.color_counts[i]; + } +} + +/* See header for documentation. */ +void rdo_decode_changed_payload( + const rdo_block_decode_state& state, + const uint8_t physical[16], + unsigned int byte_mask, + symbolic_compressed_block& block +) { + if (byte_mask & state.weight_byte_mask) + { + rdo_decode_weight_payload(state, physical, block); + } + + if (byte_mask & state.color_byte_mask) + { + rdo_decode_color_payload(state, physical, block); + } +} + +#endif diff --git a/Source/astcenc_rdo_rate.cpp b/Source/astcenc_rdo_rate.cpp new file mode 100644 index 000000000..76aadd6b7 --- /dev/null +++ b/Source/astcenc_rdo_rate.cpp @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief DEFLATE-aware token costs for the RDO entropy reduction transform. + * + * Token alphabets and extra-bit counts follow RFC 1951, sections 3.2.5 and 3.2.7. + */ + +#if !defined(ASTCENC_DECOMPRESS_ONLY) + +#include "astcenc_rdo.h" + +#include +#include +#include +#include +#include +#include + +/** @brief The base length represented by each DEFLATE length symbol. */ +static constexpr unsigned int length_base[29] { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, + 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258 +}; + +/** @brief Extra bits following each DEFLATE length symbol. */ +static constexpr unsigned int length_extra[29] { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, + 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 +}; + +/** @brief The base distance represented by each DEFLATE distance symbol. */ +static constexpr unsigned int distance_base[30] { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, + 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, + 6145, 8193, 12289, 16385, 24577 +}; + +/** @brief Extra bits following each DEFLATE distance symbol. */ +static constexpr unsigned int distance_extra[30] { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 +}; + +/** + * @brief Find the DEFLATE length symbol index. + * + * @param length The match length, in the range 3..258. + * + * @return The length symbol index, excluding the literal alphabet and end marker. + */ +static unsigned int rdo_get_length_code( + unsigned int length +) { + unsigned int code = 0; + while (((code + 1) < 29) && (length >= length_base[code + 1])) + { + code++; + } + + return code; +} + +/** + * @brief Find the DEFLATE distance symbol index. + * + * @param distance The backward distance, in the range 1..32768. + * + * @return The distance symbol index. + */ +static unsigned int rdo_get_distance_code( + unsigned int distance +) { + unsigned int code = 0; + while (((code + 1) < 30) && (distance >= distance_base[code + 1])) + { + code++; + } + + return code; +} + +/** + * @brief Hash three bytes into the local match dictionary. + * + * @param[in] data The three source bytes. + * + * @return The hash bucket index. + */ +static unsigned int rdo_hash_bytes( + const uint8_t* data +) { + unsigned int value = static_cast(data[0]) << 16; + value |= static_cast(data[1]) << 8; + value |= data[2]; + return (value * 0x1E35A7BDu) >> (32 - rdo_rate_hash_bits); +} + +/** + * @brief Build deterministic Huffman code lengths with a small unseen-symbol prior. + * + * Large bricks can exceed DEFLATE's 15-bit limit; callers then use fixed code lengths. + * + * @param[in] frequencies Token frequencies, with at most 286 entries. + * @param count The alphabet size. + * @param[out] lengths The resulting code lengths. + * + * @return True if every code length fits DEFLATE's limit. + */ +static bool rdo_build_code_lengths( + const unsigned int* frequencies, + unsigned int count, + uint8_t* lengths +) { + unsigned int weights[571]; + unsigned int parents[571]; + std::fill_n(parents, 2 * count - 1, ~0u); + for (unsigned int i = 0; i < count; i++) + { + weights[i] = frequencies[i] * 4 + 1; + } + + // Break equal-frequency ties by symbol index to keep prices invariant across platforms + auto compare = [&](unsigned int a, unsigned int b) + { + if (weights[a] != weights[b]) + { + return weights[a] > weights[b]; + } + + return a > b; + }; + + std::priority_queue, decltype(compare)> heap(compare); + for (unsigned int i = 0; i < count; i++) + { + heap.push(i); + } + + unsigned int next = count; + while (heap.size() > 1) + { + unsigned int a = heap.top(); + heap.pop(); + unsigned int b = heap.top(); + heap.pop(); + weights[next] = weights[a] + weights[b]; + parents[a] = next; + parents[b] = next; + heap.push(next++); + } + + for (unsigned int i = 0; i < count; i++) + { + unsigned int depth = 0; + for (unsigned int node = i; parents[node] != ~0u; node = parents[node]) + { + depth++; + } + + if (depth > 15) + { + return false; + } + + lengths[i] = static_cast(depth); + } + + return true; +} + +/* See header for documentation. */ +void rdo_rate_model::init( + const uint8_t* data, + unsigned int block_count, + unsigned int lookback +) { + assert(block_count <= brick_layout::block_count); + unsigned int byte_count = block_count * 16; + history_bytes = std::min(lookback, block_count) * 16; + unsigned int literal_length_frequencies[286] {}; + unsigned int distance_frequencies[30] {}; + + std::fill_n(heads, hash_size, -1); + + // Train on emitted tokens rather than raw byte frequencies + for (unsigned int position = 0; position < byte_count;) + { + unsigned int longest = 0; + unsigned int best_distance = 0; + unsigned int history_start = 0; + + if (position > history_bytes) + { + history_start = position - history_bytes; + } + + unsigned int limit = std::min(258u, byte_count - position); + if (limit >= 3) + { + int reference = heads[rdo_hash_bytes(data + position)]; + + while (reference >= static_cast(history_start)) + { + unsigned int source = static_cast(reference); + unsigned int length = 0; + + while ((length < limit) && (data[source + length] == data[position + length])) + { + length++; + } + + if (length > longest) + { + longest = length; + best_distance = position - source; + } + + reference = previous[source]; + } + } + + unsigned int consumed = 1; + if (longest >= 3) + { + literal_length_frequencies[257 + rdo_get_length_code(longest)]++; + distance_frequencies[rdo_get_distance_code(best_distance)]++; + consumed = longest; + } + else + { + literal_length_frequencies[data[position]]++; + } + + unsigned int end = position + consumed; + for (unsigned int source = position; (source < end) && ((source + 2) < byte_count); source++) + { + unsigned int hash = rdo_hash_bytes(data + source); + previous[source] = heads[hash]; + heads[hash] = static_cast(source); + } + + position = end; + } + + // Include the end marker and price unseen symbols so new trial tokens remain representable + literal_length_frequencies[256] = 1; + bool is_literal_tree_valid = rdo_build_code_lengths(literal_length_frequencies, 286, + literal_length_bits); + if (!is_literal_tree_valid) + { + for (unsigned int symbol = 0; symbol < 286; symbol++) + { + unsigned int bits = 8; + if ((symbol >= 144) && (symbol <= 255)) + { + bits = 9; + } + else if ((symbol >= 256) && (symbol <= 279)) + { + bits = 7; + } + + literal_length_bits[symbol] = static_cast(bits); + } + } + + uint8_t distance_bits[30]; + bool is_distance_tree_valid = rdo_build_code_lengths(distance_frequencies, 30, distance_bits); + if (!is_distance_tree_valid) + { + std::fill_n(distance_bits, 30, 5); + } + + // Fold Huffman lengths and DEFLATE extra bits into tables used by the inner search + for (unsigned int length = 3; length <= 16; length++) + { + unsigned int code = rdo_get_length_code(length); + length_costs[length] = static_cast(literal_length_bits[257 + code] + length_extra[code]); + } + + for (unsigned int distance = 1; distance <= max_bytes; distance++) + { + unsigned int code = rdo_get_distance_code(distance); + distance_costs[distance] = static_cast(distance_bits[code] + distance_extra[code]); + } + + // Rebuild the dictionary incrementally from accepted bytes rather than the training stream + std::fill_n(heads, hash_size, -1); + indexed_bytes = 0; +} + +/* See header for documentation. */ +void rdo_rate_model::prepare( + const uint8_t* data, + unsigned int block_index, + unsigned int history_start +) { + unsigned int block_start = block_index * 16; + unsigned int history_byte_start = history_start * 16; + std::memcpy(original, data + block_start, 16); + + // Enter only completed source substrings; accepted prefix bytes no longer change + while ((indexed_bytes + 2) < block_start) + { + unsigned int hash = rdo_hash_bytes(data + indexed_bytes); + previous[indexed_bytes] = heads[hash]; + heads[hash] = static_cast(indexed_bytes); + indexed_bytes++; + } + + // Find existing unaligned matches so unchanged bytes are not overcharged as literals + float matches[16][17]; + for (unsigned int start = 0; start < 16; start++) + { + std::fill_n(matches[start], 17, std::numeric_limits::infinity()); + unsigned int limit = 16 - start; + if (limit < 3) + { + continue; + } + + // Advance expiry per byte, including unaligned matches in the oldest history block + unsigned int first_source = history_byte_start; + if ((block_start + start) > history_bytes) + { + first_source = std::max(first_source, block_start + start - history_bytes); + } + + int reference = heads[rdo_hash_bytes(original + start)]; + while (reference >= static_cast(first_source)) + { + unsigned int source = static_cast(reference); + unsigned int max_length = std::min(limit, block_start - source); + unsigned int length = 0; + + while ((length < max_length) && (data[source + length] == original[start + length])) + { + length++; + } + + unsigned int distance = block_start + start - source; + for (unsigned int count = 3; count <= length; count++) + { + matches[start][count] = std::min(matches[start][count], get_match_cost(distance, count)); + } + + reference = previous[source]; + } + } + + // Cache the cheapest parse of every interval that a trial can leave untouched + for (unsigned int start = 0; start <= 16; start++) + { + intervals[start][start] = 0.0f; + for (unsigned int end = start + 1; end <= 16; end++) + { + float best = intervals[start][end - 1] + get_literal_cost(original[end - 1]); + for (unsigned int length = 3; length <= (end - start); length++) + { + best = std::min(best, intervals[start][end - length] + matches[end - length][length]); + } + + intervals[start][end] = best; + } + } +} + +/* See header for documentation. */ +float rdo_rate_model::get_match_cost( + unsigned int distance, + unsigned int length +) const { + assert((distance > 0) && (distance <= max_bytes)); + assert((length >= 3) && (length <= 16)); + return static_cast(length_costs[length] + distance_costs[distance]); +} + +/* See header for documentation. */ +float rdo_rate_model::get_literal_cost( + uint8_t value +) const { + return static_cast(literal_length_bits[value]); +} + +/* See header for documentation. */ +float rdo_rate_model::estimate( + const rdo_byte_match& first, + const rdo_byte_match& second +) const { + if (first.length == 0) + { + return intervals[0][16]; + } + + // Splice forced matches between cached optimal parses of the untouched intervals + const rdo_byte_match* left = &first; + const rdo_byte_match* right = &second; + if ((second.length > 0) && (second.offset < first.offset)) + { + std::swap(left, right); + } + + unsigned int end = left->offset + left->length; + float bits = intervals[0][left->offset] + left->bits; + if (second.length > 0) + { + bits += intervals[end][right->offset] + right->bits; + end = right->offset + right->length; + } + + bits += intervals[end][16]; + return bits; +} + +#endif diff --git a/Source/astcenc_rdo_scoring.cpp b/Source/astcenc_rdo_scoring.cpp new file mode 100644 index 000000000..1e2188658 --- /dev/null +++ b/Source/astcenc_rdo_scoring.cpp @@ -0,0 +1,1224 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief Fused RDO decoding and distortion for LDR blocks. + */ + +#if !defined(ASTCENC_DECOMPRESS_ONLY) + +#include + +#include "astcenc_internal.h" +#include "astcenc_rdo.h" + +/** @brief Unpacked trial endpoints shared by RGB and RGBA distortion kernels. */ +struct rdo_trial_endpoints +{ + /** @brief Low endpoints for the active partitions. */ + vint4 endpoint0[4]; + + /** @brief Differences between each partition's high and low endpoints. */ + vint4 deltas[4]; +}; + +/* See header for documentation. */ +float rdo_get_mse_limit( + float best_cost, + float rate_cost, + float mse_recip +) { + if (rate_cost >= best_cost) + { + return 0.0f; + } + + return (best_cost - rate_cost) * mse_recip; +} + +/** + * @brief Decode UNORM16 colors through the decoder's truncating FP16 intermediate. + * + * UNORM16 maps to p / 65536 with ten fraction bits, except 65535 maps to one. + * Clearing thirteen FP32 fraction bits gives the same result, including FP16 subnormals. + * + * @param color The unsigned 16-bit color values. + * + * @return The decoded LDR values in [0, 1]. + */ +static ASTCENC_SIMD_INLINE vfloat rdo_decode_ldr_color( + vint color +) { + vfloat normalized = int_to_float(color) * (1.0f / 65536.0f); + vfloat truncated = int_as_float(float_as_int(normalized) & vint(~0x1FFF)); + return select(truncated, vfloat(1.0f), color == vint(65535)); +} + +/** + * @brief Decode four UNORM16 colors through the decoder's truncating FP16 intermediate. + * + * @param color The unsigned 16-bit color values. + * + * @return The decoded LDR values in [0, 1]. + */ +static ASTCENC_SIMD_INLINE vfloat4 rdo_decode_ldr_color_4( + vint4 color +) { + vfloat4 normalized = int_to_float(color) * (1.0f / 65536.0f); + vfloat4 truncated = int_as_float(float_as_int(normalized) & vint4(~0x1FFF)); + return select(truncated, vfloat4(1.0f), color == vint4(65535)); +} + +/** + * @brief Interpolate only the batch of weights needed by the next distortion calculation. + * + * @param[in] di The fixed decimation information. + * @param[in] weights The unquantized stored weights. + * @param[in] table The prepared lookup table, used only for decimated blocks. + * @param texel_offset The first texel in the SIMD batch. + * @param plane_offset The stored weight-plane offset, either zero or WEIGHTS_PLANE2_OFFSET. + * @param max_weight_count The maximum contributing weight count for this batch. + * + * @return The integer interpolation weights for the batch. + */ +static ASTCENC_SIMD_INLINE vint rdo_interpolate_weights( + const decimation_info& di, + const uint8_t* weights, + const vtable_64x8& table, + unsigned int texel_offset, + unsigned int plane_offset, + unsigned int max_weight_count +) { + if (di.weight_count == di.texel_count) + { + return vint(weights + plane_offset + texel_offset); + } + + vint summed_value(8); + promise(max_weight_count > 0); + for (unsigned int j = 0; j < max_weight_count; j++) + { + vint texel_weights = vint(di.texel_weights_tr[j] + texel_offset) + vint(plane_offset); + vint texel_weights_int(di.texel_weight_contribs_int_tr[j] + texel_offset); + summed_value += vtable_lookup_32bit(table, texel_weights) * texel_weights_int; + } + + return lsr<4>(summed_value); +} + +/** + * @brief Reuse original cached weights or interpolate a changed trial payload. + * + * @param[in] di The fixed decimation information. + * @param[in] weights The trial's unquantized stored weights. + * @param[in] table The prepared lookup table, used only for decimated blocks. + * @param texel_offset The first texel in the SIMD batch. + * @param plane_offset The stored weight-plane offset, either zero or WEIGHTS_PLANE2_OFFSET. + * @param[in] cache The initialized original reconstruction cache. + * + * @return The integer interpolation weights for the batch. + */ +static ASTCENC_SIMD_INLINE vint rdo_interpolate_weights( + const decimation_info& di, + const uint8_t* weights, + const vtable_64x8& table, + unsigned int texel_offset, + unsigned int plane_offset, + const rdo_block_difference_cache& cache +) { + if (cache.is_weights_unchanged) + { + unsigned int plane = 0; + if (plane_offset != 0) + { + plane = 1; + } + + return vint(cache.weights[plane] + texel_offset); + } + + return rdo_interpolate_weights(di, weights, table, texel_offset, plane_offset, + cache.weight_counts[texel_offset / ASTCENC_SIMD_WIDTH]); +} + +/* See header for documentation. */ +void rdo_prepare_difference_cache( + const astcenc_config& config, + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + rdo_block_difference_cache& cache +) { + assert(block.block_type == SYM_BTYPE_NONCONST); + static_assert(BLOCK_MAX_TEXELS <= 224); + static_assert(ASTCENC_SIMD_WIDTH >= 4); + + // Unchanged payloads can reuse reconstruction, independent of the bytes copied by a trial + for (unsigned int p = 0; p < block.partition_count; p++) + { + vint4 endpoint0; + vint4 endpoint1; + bool is_rgb_lns; + bool is_alpha_lns; + [[maybe_unused]] bool is_valid = unpack_color_endpoints( + config.profile, block.color_formats[p], block.color_values[p], + is_rgb_lns, is_alpha_lns, endpoint0, endpoint1); + assert(is_valid); + + store(endpoint0, cache.endpoints[p][0]); + store(endpoint1, cache.endpoints[p][1]); + } + + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& di = bsd.get_decimation_info(mode.decimation_mode); + unsigned int plane_count = 1; + if (mode.is_dual_plane) + { + plane_count = 2; + } + + // Contributor counts depend only on the fixed layout, so reduce them once for all trials + for (unsigned int i = 0; i < bsd.texel_count; i += ASTCENC_SIMD_WIDTH) + { + cache.weight_counts[i / ASTCENC_SIMD_WIDTH] = + static_cast(hmax_s(vint(di.texel_weight_count + i))); + } + + vtable_64x8 table; + if (di.weight_count != bsd.texel_count) + { + vtable_prepare(table, block.weights); + } + + for (unsigned int plane = 0; plane < plane_count; plane++) + { + unsigned int plane_offset = plane * WEIGHTS_PLANE2_OFFSET; + if (di.weight_count == bsd.texel_count) + { + std::memcpy(cache.weights[plane], block.weights + plane_offset, bsd.texel_count); + size_t padded_count = round_up_to_simd_multiple_vla(bsd.texel_count); + std::memset(cache.weights[plane] + bsd.texel_count, 0, padded_count - bsd.texel_count); + continue; + } + + for (unsigned int i = 0; i < bsd.texel_count; i += ASTCENC_SIMD_WIDTH) + { + vint weights = rdo_interpolate_weights(di, block.weights, table, i, plane_offset, + cache.weight_counts[i / ASTCENC_SIMD_WIDTH]); + pack_and_store_low_bytes(weights, cache.weights[plane] + i); + } + } + + size_t padded_count = round_up_to_simd_multiple_vla(bsd.texel_count); + std::fill_n(cache.texel_errors + bsd.texel_count, padded_count - bsd.texel_count, 0.0f); + + cache.is_colors_unchanged = true; + cache.is_weights_unchanged = true; +} + +/* See header for documentation. */ +void rdo_prepare_difference_source( + image_block& original, + unsigned int texel_count +) { + for (unsigned int i = 0; i < texel_count; i++) + { + vfloat4 texel = original.texel(i) * (255.0f / 65535.0f); + original.data_rgba[i] = texel; + original.data_r[i] = texel.lane<0>(); + original.data_g[i] = texel.lane<1>(); + original.data_b[i] = texel.lane<2>(); + original.data_a[i] = texel.lane<3>(); + } + + // Initialize inactive source lanes once per block for a padded final SIMD batch + size_t padded_count = round_up_to_simd_multiple_vla(texel_count); + for (size_t i = texel_count; i < padded_count; i++) + { + original.data_r[i] = 0.0f; + original.data_g[i] = 0.0f; + original.data_b[i] = 0.0f; + original.data_a[i] = 0.0f; + } + + original.data_mean = original.data_mean * (255.0f / 65535.0f); + original.data_min = original.data_min * (255.0f / 65535.0f); + original.data_max = original.data_max * (255.0f / 65535.0f); +} + +/* See header for documentation. */ +rdo_block_statistics rdo_compute_original_difference( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + rdo_block_difference_cache& cache +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& partition = bsd.get_partition_info(block.partition_count, block.partition_index); + vmask4 plane2_mask = vint4::lane_id() == vint4(block.plane2_component); + vint4 endpoints[4]; + vint4 deltas[4]; + + for (unsigned int p = 0; p < block.partition_count; p++) + { + endpoints[p] = vint4(cache.endpoints[p][0]); + deltas[p] = vint4(cache.endpoints[p][1]) - endpoints[p]; + } + + // Reconstruct directly into the error cache without an intermediate decoded image block + for (unsigned int i = 0; i < bsd.texel_count; i++) + { + unsigned int p = partition.partition_of_texel[i]; + vint4 weight(cache.weights[0][i]); + if (mode.is_dual_plane) + { + weight = select(weight, vint4(cache.weights[1][i]), plane2_mask); + } + + vint4 color = endpoints[p] + asr<6>(deltas[p] * weight + vint4(32)); + vfloat4 error = original.texel(i) - rdo_decode_ldr_color_4(color) * 255.0f; + cache.texel_errors[i] = dot_s(error * error, original.channel_weight); + } + + vfloat weight_r(original.channel_weight.lane<0>()); + vfloat weight_g(original.channel_weight.lane<1>()); + vfloat weight_b(original.channel_weight.lane<2>()); + vfloat weight_a(original.channel_weight.lane<3>()); + + vfloat mean_r(original.data_mean.lane<0>()); + vfloat mean_g(original.data_mean.lane<1>()); + vfloat mean_b(original.data_mean.lane<2>()); + vfloat mean_a(original.data_mean.lane<3>()); + + vfloatacc squared_error = vfloatacc::zero(); + vfloatacc variance = vfloatacc::zero(); + + // Image edges use the source loader's replicated texels across the complete block footprint + for (unsigned int i = 0; i < bsd.texel_count; i += ASTCENC_SIMD_WIDTH) + { + vmask active = (vint(i) + vint::lane_id()) < vint(bsd.texel_count); + haccumulate(squared_error, vfloat(cache.texel_errors + i), active); + + vfloat deviation_r = vfloat(original.data_r + i) - mean_r; + vfloat deviation_g = vfloat(original.data_g + i) - mean_g; + vfloat deviation_b = vfloat(original.data_b + i) - mean_b; + vfloat deviation_a = vfloat(original.data_a + i) - mean_a; + + vfloat metric_r = deviation_r * deviation_r * weight_r; + vfloat metric_g = deviation_g * deviation_g * weight_g; + vfloat metric_b = deviation_b * deviation_b * weight_b; + vfloat metric_a = deviation_a * deviation_a * weight_a; + + haccumulate(variance, max(max(metric_r, metric_b), max(metric_g, metric_a)), active); + } + + float texel_count = static_cast(bsd.texel_count); + + return { + .mse = hadd_s(squared_error) / texel_count, + .std_dev = std::sqrt(hadd_s(variance) / texel_count) + }; +} + +/** + * @brief Unpack trial endpoints and identify an exactly zero alpha error. + * + * @param[in] config The LDR compressor configuration. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param[in] cache The initialized original reconstruction cache. + * @param[out] endpoints The prepared trial endpoints and spans. + * + * @return True if every endpoint and source texel has exactly opaque alpha. + */ +static bool rdo_prepare_trial_endpoints( + const astcenc_config& config, + const symbolic_compressed_block& block, + const image_block& original, + const rdo_block_difference_cache& cache, + rdo_trial_endpoints& endpoints +) { + bool is_rgb = (original.data_min.lane<3>() == 255.0f) + && (original.data_max.lane<3>() == 255.0f); + for (unsigned int p = 0; p < block.partition_count; p++) + { + vint4 endpoint0; + vint4 endpoint1; + if (cache.is_colors_unchanged) + { + endpoint0 = vint4(cache.endpoints[p][0]); + endpoint1 = vint4(cache.endpoints[p][1]); + } + else + { + bool is_rgb_lns; + bool is_alpha_lns; + [[maybe_unused]] bool is_valid = unpack_color_endpoints( + config.profile, block.color_formats[p], block.color_values[p], + is_rgb_lns, is_alpha_lns, endpoint0, endpoint1); + assert(is_valid); + } + + endpoints.endpoint0[p] = endpoint0; + endpoints.deltas[p] = endpoint1 - endpoint0; + is_rgb = is_rgb && (endpoint0.lane<3>() == 65535) && (endpoint1.lane<3>() == 65535); + } + + return is_rgb; +} + +/** + * @brief Score a dual-plane RGB block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_2plane_rgb( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vmask plane2_r(block.plane2_component == 0); + vmask plane2_g(block.plane2_component == 1); + vmask plane2_b(block.plane2_component == 2); + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + vint weight2 = rdo_interpolate_weights(decimation, block.weights, table, i, + WEIGHTS_PLANE2_OFFSET, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + is_batch_unchanged = is_batch_unchanged && all(weight2 == vint(cache.weights[1] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + + weight_r = select(weight, weight2, plane2_r); + weight_g = select(weight, weight2, plane2_g); + weight_b = select(weight, weight2, plane2_b); + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = error_r * error_r * channel_weight_r + + error_b * error_b * channel_weight_b + + error_g * error_g * channel_weight_g; + + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/** + * @brief Score a dual-plane RGBA block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_2plane_rgba( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vmask plane2_r(block.plane2_component == 0); + vmask plane2_g(block.plane2_component == 1); + vmask plane2_b(block.plane2_component == 2); + vmask plane2_a(block.plane2_component == 3); + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + vint endpoint_a(endpoint0.lane<3>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + vint delta_a(endpoint_delta.lane<3>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + vfloat channel_weight_a(original.channel_weight.lane<3>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + vint weight2 = rdo_interpolate_weights(decimation, block.weights, table, i, + WEIGHTS_PLANE2_OFFSET, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + is_batch_unchanged = is_batch_unchanged && all(weight2 == vint(cache.weights[1] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + vint weight_a = weight; + + weight_r = select(weight, weight2, plane2_r); + weight_g = select(weight, weight2, plane2_g); + weight_b = select(weight, weight2, plane2_b); + weight_a = select(weight, weight2, plane2_a); + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + vint data_a = endpoint_a; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + vint texel_delta_a = delta_a; + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + vint color_a = data_a + asr<6>(texel_delta_a * weight_a + vint(32)); + + vfloat error_a = loada(original.data_a + i) - rdo_decode_ldr_color(color_a) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = (error_r * error_r * channel_weight_r + error_b * error_b * channel_weight_b) + + (error_g * error_g * channel_weight_g + error_a * error_a * channel_weight_a); + + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/** + * @brief Score a multipartition RGB block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_1plane_rgb( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + + const auto& partition = bsd.get_partition_info(block.partition_count, block.partition_index); + vint partitions(partition.partition_of_texel + i); + for (unsigned int p = 1; p < block.partition_count; p++) + { + vmask selected = partitions == vint(p); + data_r = select(data_r, vint(endpoints.endpoint0[p].lane<0>()), selected); + data_g = select(data_g, vint(endpoints.endpoint0[p].lane<1>()), selected); + data_b = select(data_b, vint(endpoints.endpoint0[p].lane<2>()), selected); + texel_delta_r = select(texel_delta_r, vint(endpoints.deltas[p].lane<0>()), selected); + texel_delta_g = select(texel_delta_g, vint(endpoints.deltas[p].lane<1>()), selected); + texel_delta_b = select(texel_delta_b, vint(endpoints.deltas[p].lane<2>()), selected); + } + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = (error_r * error_r * channel_weight_r + error_b * error_b * channel_weight_b) + + error_g * error_g * channel_weight_g; + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/** + * @brief Score a multipartition RGBA block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_1plane_rgba( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + vint endpoint_a(endpoint0.lane<3>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + vint delta_a(endpoint_delta.lane<3>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + vfloat channel_weight_a(original.channel_weight.lane<3>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + vint weight_a = weight; + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + vint data_a = endpoint_a; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + vint texel_delta_a = delta_a; + + const auto& partition = bsd.get_partition_info(block.partition_count, block.partition_index); + vint partitions(partition.partition_of_texel + i); + for (unsigned int p = 1; p < block.partition_count; p++) + { + vmask selected = partitions == vint(p); + data_r = select(data_r, vint(endpoints.endpoint0[p].lane<0>()), selected); + data_g = select(data_g, vint(endpoints.endpoint0[p].lane<1>()), selected); + data_b = select(data_b, vint(endpoints.endpoint0[p].lane<2>()), selected); + texel_delta_r = select(texel_delta_r, vint(endpoints.deltas[p].lane<0>()), selected); + texel_delta_g = select(texel_delta_g, vint(endpoints.deltas[p].lane<1>()), selected); + texel_delta_b = select(texel_delta_b, vint(endpoints.deltas[p].lane<2>()), selected); + data_a = select(data_a, vint(endpoints.endpoint0[p].lane<3>()), selected); + texel_delta_a = select(texel_delta_a, vint(endpoints.deltas[p].lane<3>()), selected); + } + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + vint color_a = data_a + asr<6>(texel_delta_a * weight_a + vint(32)); + + vfloat error_a = loada(original.data_a + i) - rdo_decode_ldr_color(color_a) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = (error_r * error_r * channel_weight_r + error_b * error_b * channel_weight_b) + + (error_g * error_g * channel_weight_g + error_a * error_a * channel_weight_a); + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/** + * @brief Score a single-partition RGB block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_1plane_1partition_rgb( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = (error_r * error_r * channel_weight_r + error_b * error_b * channel_weight_b) + + error_g * error_g * channel_weight_g; + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/** + * @brief Score a single-partition RGBA block using SIMD across texels. + * + * @param[in] bsd The block size descriptor. + * @param[in] block The nonconstant symbolic encoding. + * @param[in] original The prepared source texels and normalized channel weights. + * @param mse_limit The MSE bound, or infinity for a complete score. + * @param[in] cache The initialized original reconstruction cache. + * @param[in] endpoints The prepared trial endpoints and spans. + * + * @return Weighted MSE, or maximum MSE for losing trials. + */ +static ASTCENC_SIMD_INLINE float rdo_compute_difference_1plane_1partition_rgba( + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache, + const rdo_trial_endpoints& endpoints +) { + const auto& mode = bsd.get_block_mode(block.block_mode); + const auto& decimation = bsd.get_decimation_info(mode.decimation_mode); + vint4 endpoint0 = endpoints.endpoint0[0]; + vint4 endpoint_delta = endpoints.deltas[0]; + + vtable_64x8 table; + if ((decimation.weight_count != bsd.texel_count) && !cache.is_weights_unchanged) + { + vtable_prepare(table, block.weights); + } + + vint endpoint_r(endpoint0.lane<0>()); + vint endpoint_g(endpoint0.lane<1>()); + vint endpoint_b(endpoint0.lane<2>()); + vint endpoint_a(endpoint0.lane<3>()); + + vint delta_r(endpoint_delta.lane<0>()); + vint delta_g(endpoint_delta.lane<1>()); + vint delta_b(endpoint_delta.lane<2>()); + vint delta_a(endpoint_delta.lane<3>()); + + vfloat channel_weight_r(original.channel_weight.lane<0>()); + vfloat channel_weight_g(original.channel_weight.lane<1>()); + vfloat channel_weight_b(original.channel_weight.lane<2>()); + vfloat channel_weight_a(original.channel_weight.lane<3>()); + + unsigned int texel_count = bsd.texel_count; + size_t vector_end = round_up_to_simd_multiple_vla(texel_count); + + vfloatacc squared_error = vfloatacc::zero(); + float error_limit = mse_limit * static_cast(texel_count); + unsigned int i = 0; + + // Reconstruct only the next SIMD batch so losing trials can stop before a full block decode + for (; i < vector_end; i += ASTCENC_SIMD_WIDTH) + { + vint weight = rdo_interpolate_weights(decimation, block.weights, table, i, 0, cache); + + // Reuse cached errors when a weight-only change leaves this entire batch unchanged + bool is_batch_unchanged = cache.is_colors_unchanged; + if (is_batch_unchanged) + { + is_batch_unchanged = all(weight == vint(cache.weights[0] + i)); + } + + if (is_batch_unchanged) + { + haccumulate(squared_error, vfloat(cache.texel_errors + i)); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + + continue; + } + + vint weight_r = weight; + vint weight_g = weight; + vint weight_b = weight; + vint weight_a = weight; + + vint data_r = endpoint_r; + vint data_g = endpoint_g; + vint data_b = endpoint_b; + vint data_a = endpoint_a; + + vint texel_delta_r = delta_r; + vint texel_delta_g = delta_g; + vint texel_delta_b = delta_b; + vint texel_delta_a = delta_a; + + vint color_r = data_r + asr<6>(texel_delta_r * weight_r + vint(32)); + vint color_g = data_g + asr<6>(texel_delta_g * weight_g + vint(32)); + vint color_b = data_b + asr<6>(texel_delta_b * weight_b + vint(32)); + + vfloat error_r = loada(original.data_r + i) + - rdo_decode_ldr_color(color_r) * 255.0f; + vfloat error_g = loada(original.data_g + i) + - rdo_decode_ldr_color(color_g) * 255.0f; + vfloat error_b = loada(original.data_b + i) + - rdo_decode_ldr_color(color_b) * 255.0f; + vint color_a = data_a + asr<6>(texel_delta_a * weight_a + vint(32)); + + vfloat error_a = loada(original.data_a + i) - rdo_decode_ldr_color(color_a) * 255.0f; + + // Preserve channel reduction order across all vector widths + vfloat metric = (error_r * error_r * channel_weight_r + error_b * error_b * channel_weight_b) + + (error_g * error_g * channel_weight_g + error_a * error_a * channel_weight_a); + // Exclude inactive texels from the invariant SIMD reduction + if ((i + ASTCENC_SIMD_WIDTH) > texel_count) + { + vmask active = (vint(i) + vint::lane_id()) < vint(texel_count); + metric = select(vfloat(0.0f), metric, active); + } + + haccumulate(squared_error, metric); + + if (hadd_s(squared_error) >= error_limit) + { + return std::numeric_limits::max(); + } + } + + return hadd_s(squared_error) / static_cast(texel_count); +} + +/* See header for documentation. */ +float rdo_compute_block_difference( + const astcenc_config& config, + const block_size_descriptor& bsd, + const symbolic_compressed_block& block, + const image_block& original, + float mse_limit, + const rdo_block_difference_cache& cache +) { + assert(block.block_type == SYM_BTYPE_NONCONST); + rdo_trial_endpoints endpoints; + bool is_rgb = rdo_prepare_trial_endpoints(config, block, original, cache, endpoints); + + // The core compressor emits dual-plane blocks only with a single partition + if (block.partition_count > 1) + { + assert(!bsd.get_block_mode(block.block_mode).is_dual_plane); + if (is_rgb) + { + return rdo_compute_difference_1plane_rgb( + bsd, block, original, mse_limit, cache, endpoints); + } + + return rdo_compute_difference_1plane_rgba( + bsd, block, original, mse_limit, cache, endpoints); + } + + const auto& mode = bsd.get_block_mode(block.block_mode); + if (mode.is_dual_plane) + { + if (is_rgb) + { + return rdo_compute_difference_2plane_rgb( + bsd, block, original, mse_limit, cache, endpoints); + } + + return rdo_compute_difference_2plane_rgba( + bsd, block, original, mse_limit, cache, endpoints); + } + + if (is_rgb) + { + return rdo_compute_difference_1plane_1partition_rgb( + bsd, block, original, mse_limit, cache, endpoints); + } + + return rdo_compute_difference_1plane_1partition_rgba( + bsd, block, original, mse_limit, cache, endpoints); +} + +#endif diff --git a/Source/astcenc_rdo_search.cpp b/Source/astcenc_rdo_search.cpp new file mode 100644 index 000000000..410f07ed7 --- /dev/null +++ b/Source/astcenc_rdo_search.cpp @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @file + * @brief RDO fixed-offset byte search using DEFLATE-aware rate estimates. + * + * Based on Rich Geldreich's public-domain ERT implementation in bc7enc_rdo. + * See https://github.com/richgel999/bc7enc_rdo/blob/master/ert.cpp. + */ + +#if !defined(ASTCENC_DECOMPRESS_ONLY) + +#include "astcenc_internal.h" +#include "astcenc_rdo.h" + +#include +#include +#include +#include +#include +#include + +/** @brief The physical block size in bytes. */ +static constexpr unsigned int block_size = 16; + +/** @brief The minimum LZ match length in bytes. */ +static constexpr unsigned int min_match_length = 3; + +static_assert((brick_layout::block_count > 0) && (brick_layout::block_count <= 256), + "RDO block tags must fit in eight bits"); + +static_assert(brick_layout::block_count <= std::numeric_limits::max() / block_size, + "RDO byte offsets must fit in a signed int"); + +/** @brief Shared scoring inputs and the best result for one block's byte searches. */ +struct rdo_search_state +{ + /** @brief The original physical bytes, unchanged until both searches finish. */ + const uint8_t* original; + + /** @brief The distortion and history settings. */ + const rdo_parameters& params; + + /** @brief Rate estimates prepared for the current block. */ + const rdo_rate_model& rates; + + /** @brief Source data and reconstruction caches used for distortion scoring. */ + rdo_brick_context& ctx; + + /** @brief The current block's index in the compact brick stream. */ + unsigned int block_index; + + /** @brief The original block's estimated storage cost in bits. */ + float original_rate; + + /** @brief The distortion multiplier protecting smooth blocks. */ + float mse_scale; + + /** @brief The reciprocal of the distortion multiplier. */ + float mse_recip; + + /** @brief The maximum trial MSE allowed by the RMS guard. */ + float max_mse; + + /** @brief The lowest rate-distortion cost found so far. */ + float best_cost; + + /** @brief The physical bytes producing the best cost. */ + uint8_t best_block[block_size]; +}; + +/** + * @brief Build byte prefixes for selecting a copied interval. + * + * @return The masks for prefixes of zero to sixteen bytes. + */ +static constexpr std::array, 17> rdo_make_prefix_masks() +{ + std::array, 17> masks {}; + for (unsigned int end = 0; end <= 16; end++) + { + for (unsigned int i = 0; i < end; i++) + { + masks[end][i] = 0xFF; + } + } + + return masks; +} + +/** @brief Byte prefixes used to select a copied interval without a variable-size memcpy. */ +static constexpr auto prefix_masks = rdo_make_prefix_masks(); + +/** @brief Estimated cost of continuing the previous block's match, in bits. */ +static constexpr float continuation_bits = 1.0f; + +/** @brief Polynomial base for reusable substring hashes. */ +static constexpr uint32_t hash_base = 16777619u; + +/** + * @brief Build powers for removing a substring's preceding bytes. + * + * @return The polynomial base raised to powers zero through sixteen. + */ +static constexpr std::array rdo_make_hash_powers() +{ + std::array powers {}; + powers[0] = 1; + for (unsigned int i = 1; i <= 16; i++) + { + powers[i] = powers[i - 1] * hash_base; + } + + return powers; +} + +/** @brief Polynomial factors for extracting each substring length. */ +static constexpr auto hash_powers = rdo_make_hash_powers(); + +/* See header for documentation. */ +void rdo_prepare_substring_hashes( + const uint8_t data[16], + uint32_t prefixes[17] +) { + prefixes[0] = 0; + for (unsigned int i = 0; i < block_size; i++) + { + prefixes[i + 1] = prefixes[i] * hash_base + data[i] + 1; + } +} + +/* See header for documentation. */ +void rdo_hash_substrings( + const uint32_t* prefixes, + unsigned int length, + uint32_t hashes[16] +) { + assert((length >= min_match_length) && (length <= block_size)); + unsigned int count = block_size - length + 1; + for (unsigned int offset = 0; offset < count; offset += ASTCENC_SIMD_WIDTH) + { + vint starts(reinterpret_cast(prefixes + offset)); + vint ends(reinterpret_cast(prefixes + offset + length)); + vint hash = ends - starts * vint(static_cast(hash_powers[length])); + vint offsets = vint(offset) + vint::lane_id(); + hash += offsets * vint(0x1E35A7BD) + vint(length); + hash = (hash ^ lsr<16>(hash)) * vint(0x7FEB352D); + hash = (hash ^ lsr<15>(hash)) * vint(0x2C1B3C6D); + hash = hash ^ lsr<16>(hash); + store_lanes_masked(reinterpret_cast(hashes + offset), hash, offsets < vint(count)); + } +} + +/** + * @brief Find bytes whose replacement would change protected layout bits. + * + * @param[in] original The current target block. + * @param[in] source The predecessor block supplying copied bytes. + * @param[in] protected_mask The target block's protected bits. + * + * @return A bit mask with one bit for each incompatible source byte. + */ +static uint32_t rdo_get_incompatible_bytes( + const uint8_t original[16], + const uint8_t source[16], + const uint8_t protected_mask[16] +) { + static_assert((block_size % ASTCENC_SIMD_WIDTH) == 0); + uint32_t incompatible = 0; + for (unsigned int i = 0; i < block_size; i += ASTCENC_SIMD_WIDTH) + { + vint changed = (vint(original + i) ^ vint(source + i)) & vint(protected_mask + i); + incompatible |= mask(changed != vint(0)) << i; + } + + return incompatible; +} + +/** + * @brief Test whether a trial exceeds the physical bit-change search budget. + * + * @param[in] original The original physical block. + * @param[in] trial The trial physical block. + * + * @return True if the configured bit-change budget is exceeded. + */ +static bool rdo_is_distant_trial( + const uint8_t original[16], + const uint8_t trial[16] +) { + uint64_t original_words[2]; + uint64_t trial_words[2]; + std::memcpy(original_words, original, sizeof(original_words)); + std::memcpy(trial_words, trial, sizeof(trial_words)); + unsigned int changed_bits = std::popcount(original_words[0] ^ trial_words[0]) + + std::popcount(original_words[1] ^ trial_words[1]); + return changed_bits > rdo_max_changed_bits; +} + +/** @brief Padded row offsets for each cached substring length. */ +static constexpr unsigned int substring_offsets[17] { + 0, 0, 0, 0, 16, 32, 44, 56, 68, 80, 88, 96, 104, 112, 116, 120, 124 +}; + +/** + * @brief Score a trial and retain it if it improves the block's rate-distortion cost. + * + * @param[in,out] search The current block's scoring state and best result. + * @param[in] trial The candidate physical bytes. + * @param rate The candidate's estimated storage cost in bits. + * @param byte_mask The physical bytes that may differ from the original. + * + * @return True if the trial improves the best cost. + */ +static bool rdo_evaluate_trial( + rdo_search_state& search, + const uint8_t trial[block_size], + float rate, + unsigned int byte_mask +) { + // Spend distortion work only on trials predicted to reduce storage + if (rate >= search.original_rate) + { + return false; + } + + // Convert remaining cost headroom into a distortion cutoff + float rate_cost = rate * search.params.lambda; + float mse_limit = std::min(search.max_mse, + rdo_get_mse_limit(search.best_cost, rate_cost, search.mse_recip)); + if (mse_limit <= 0.0f) + { + return false; + } + + // Large physical changes rarely preserve useful endpoints and weights + if (rdo_is_distant_trial(search.original, trial)) + { + return false; + } + + float trial_mse = rdo_block_difference(search.ctx, trial, byte_mask, mse_limit); + if (trial_mse >= search.max_mse) + { + return false; + } + + float cost = trial_mse * search.mse_scale + rate_cost; + if (!(cost < search.best_cost)) + { + return false; + } + + search.best_cost = cost; + std::memcpy(search.best_block, trial, block_size); + return true; +} + +/** + * @brief Search predecessor blocks for the best single copied byte range. + * + * @param[in,out] search The current block's scoring state and best result. + * @param[in] data The compact brick stream with finalized predecessors. + * @param[in] mask The current block's protected layout bits. + * @param search_start The first predecessor to search. + * @param continuation_source The next source byte of a preceding match, or -1. + * @param[in] source_hashes Cached substring hashes for finalized predecessors. + * @param[in,out] hashes Payload deduplication entries tagged by block index. + * @param[out] incompatible Incompatible byte masks for searched predecessors. + * + * @return The best first match, or an empty match if no trial improves the cost. + */ +static rdo_byte_match rdo_search_first_match( + rdo_search_state& search, + const uint8_t* data, + const uint8_t mask[block_size], + unsigned int search_start, + int continuation_source, + const uint32_t source_hashes[][128], + uint32_t hashes[rdo_trial_hash_entries], + uint32_t incompatible[brick_layout::block_count] +) { + rdo_byte_match first_match; + vint4 original_bytes = vint4::load(search.original); + unsigned int block_tag = search.block_index; + + // Copy aligned byte ranges from finalized predecessors, trying long matches first + for (unsigned int previous = search.block_index; previous > search_start;) + { + previous--; + const uint8_t* source = data + previous * block_size; + vint4 changed_bytes = vint4::load(source) ^ original_bytes; + uint32_t invalid_bytes = rdo_get_incompatible_bytes(search.original, source, mask); + incompatible[previous] = invalid_bytes; + unsigned int distance = (search.block_index - previous) * block_size; + // Intersect byte compatibility to enumerate only valid starts for each match length + uint32_t compatible = (~invalid_bytes) & 0xFFFFu; + uint32_t valid_offsets[17]; + uint32_t valid = compatible; + for (unsigned int length = 1; length <= block_size; length++) + { + valid_offsets[length] = valid; + valid &= compatible >> length; + } + + for (unsigned int length = block_size; length >= min_match_length; length--) + { + uint32_t offsets = valid_offsets[length]; + if (offsets == 0) + { + continue; + } + + float bits = search.rates.get_match_cost(distance, length); + uint32_t match_mask = (1u << length) - 1; + const uint32_t* substring_hashes = source_hashes[previous] + substring_offsets[length]; + + for (; offsets != 0; offsets &= offsets - 1) + { + unsigned int offset = std::countr_zero(offsets); + unsigned int source_offset = previous * block_size + offset; + float trial_bits = bits; + uint32_t hash = substring_hashes[offset]; + uint32_t hash_entry = (hash & 0xFFFFFF00) | block_tag; + if ((static_cast(source_offset) == continuation_source) && (offset == 0)) + { + trial_bits = continuation_bits; + } + else + { + // Deduplicate payloads except continuations with a cheaper match price + uint32_t entry = hashes[hash & (rdo_trial_hash_entries - 1)]; + if (entry == hash_entry) + { + continue; + } + } + + hashes[hash & (rdo_trial_hash_entries - 1)] = hash_entry; + + uint8_t trial[block_size]; + vint4 copied_bytes = vint4::load(prefix_masks[offset + length].data()) + ^ vint4::load(prefix_masks[offset].data()); + store(original_bytes ^ (changed_bytes & copied_bytes), trial); + float rate = search.rates.estimate({ length, source_offset, offset, trial_bits }, {}); + if (rdo_evaluate_trial(search, trial, rate, match_mask << offset)) + { + first_match = { length, source_offset, offset, trial_bits }; + } + } + } + } + + return first_match; +} + +/** + * @brief Hold the best first match fixed and search for a second disjoint byte range. + * + * @param[in,out] search The current block's scoring state and best result. + * @param[in] data The compact brick stream with finalized predecessors. + * @param search_start The first predecessor to search. + * @param[in] first_match The best single copied range. + * @param[in] incompatible Incompatible byte masks from the first search. + * + * @return The best second match, or an empty match if no trial improves the cost. + */ +static rdo_byte_match rdo_search_second_match( + rdo_search_state& search, + const uint8_t* data, + unsigned int search_start, + const rdo_byte_match& first_match, + const uint32_t incompatible[brick_layout::block_count] +) { + rdo_byte_match second_match; + if ((!rdo_is_two_matches_enabled) || (first_match.length == 0) + || (first_match.length > block_size - min_match_length)) + { + return second_match; + } + + uint8_t first_block[block_size]; + std::memcpy(first_block, search.best_block, block_size); + vint4 first_bytes = vint4::load(first_block); + float first_rate = search.rates.estimate(first_match, {}); + unsigned int first_mask = ((1u << first_match.length) - 1) << first_match.offset; + for (unsigned int previous = search.block_index; previous > search_start;) + { + previous--; + const uint8_t* source = data + previous * block_size; + vint4 changed_bytes = vint4::load(source) ^ first_bytes; + uint32_t invalid_bytes = incompatible[previous]; + unsigned int distance = (search.block_index - previous) * block_size; + for (unsigned int length = min_match_length; + length <= block_size - first_match.length; length++) + { + float bits = search.rates.get_match_cost(distance, length); + uint32_t match_mask = (1u << length) - 1; + + for (unsigned int offset = 0; offset <= block_size - length; offset++) + { + if ((offset < first_match.offset + first_match.length) + && (offset + length > first_match.offset)) + { + continue; + } + + if ((invalid_bytes & (match_mask << offset)) != 0) + { + continue; + } + + uint8_t trial[block_size]; + vint4 copied_bytes = vint4::load(prefix_masks[offset + length].data()) + ^ vint4::load(prefix_masks[offset].data()); + store(first_bytes ^ (changed_bytes & copied_bytes), trial); + rdo_byte_match match { length, previous * block_size + offset, offset, bits }; + float trial_rate = search.rates.estimate(first_match, match); + // Require meaningful estimated savings before decoding second matches + if ((first_rate - trial_rate) < rdo_min_second_match_saving) + { + continue; + } + + unsigned int byte_mask = first_mask | (match_mask << offset); + if (rdo_evaluate_trial(search, trial, trial_rate, byte_mask)) + { + second_match = { length, previous * block_size + offset, offset, 0.0f }; + } + } + } + } + + return second_match; +} + +/* See header for documentation. */ +void rdo_reduce_entropy( + rdo_brick_context& ctx, + uint8_t* data, + unsigned int block_count, + const uint8_t* masks, + const rdo_parameters& params +) { + assert(block_count <= brick_layout::block_count); + + // Learn brick-wide token prices before optimizing the accepted prefix in stream order + rdo_rate_model rates; + rates.init(data, block_count, rdo_history_bytes / 16); + + constexpr unsigned int hash_size = rdo_trial_hash_entries; + uint32_t hashes[hash_size] {}; + uint32_t source_hashes[brick_layout::block_count][128]; + uint32_t incompatible[brick_layout::block_count]; + int continuation_source = -1; + + for (unsigned int block_index = 0; block_index < block_count; block_index++) + { + if (block_index > 0) + { + // A predecessor's bytes remain final, so its substring hashes can serve every later block + // Zero padding permits full-width prefix loads for the masked final hash batch + uint32_t prefixes[16 + ASTCENC_SIMD_WIDTH] {}; + rdo_prepare_substring_hashes(data + (block_index - 1) * block_size, prefixes); + for (unsigned int length = min_match_length; length <= block_size; length++) + { + rdo_hash_substrings(prefixes, length, source_hashes[block_index - 1] + substring_offsets[length]); + } + } + + uint8_t* original = data + block_index * block_size; + + // Refresh token prices once optimized history covers half of the brick + if (rdo_is_midpoint_refresh_enabled && (block_index > 0) && (block_index == block_count / 2)) + { + rates.init(data, block_count, rdo_history_bytes / 16); + } + + // Keep constant blocks in byte history without spending work on their distortion or trials + if ((original[0] == 0xFC) && ((original[1] & 1) != 0)) + { + continuation_source = -1; + continue; + } + + const uint8_t* mask = masks + block_index * block_size; + const auto statistics = rdo_initialize_block(ctx, block_index); + + // Penalize distortion more strongly in smooth regions where small errors are more visible + float smooth = std::clamp(statistics.std_dev / rdo_max_smooth_std_dev, 0.0f, 1.0f); + smooth *= smooth; + float mse_scale = rdo_smooth_mse_scale + (1.0f - rdo_smooth_mse_scale) * smooth; + float mse_recip = 1.0f / mse_scale; + + // Rate scoring retains the full window even when speculative copies search fewer blocks + unsigned int history_count = std::min(block_index, rdo_history_bytes / 16); + unsigned int history_start = block_index - history_count; + unsigned int search_start = block_index - std::min(history_count, params.search_lookback); + + rates.prepare(data, block_index, history_start); + float original_rate = rates.estimate({}, {}); + + float original_cost = statistics.mse * mse_scale + original_rate * params.lambda; + + // Preserve the original RMS guard, including its error floor for exact blocks + float max_mse = rdo_max_rms_increase * rdo_max_rms_increase + * std::max(statistics.mse, 1.0f); + + rdo_search_state search { + .original = original, + .params = params, + .rates = rates, + .ctx = ctx, + .block_index = block_index, + .original_rate = original_rate, + .mse_scale = mse_scale, + .mse_recip = mse_recip, + .max_mse = max_mse, + .best_cost = original_cost, + .best_block = {} + }; + + std::memcpy(search.best_block, original, block_size); + + rdo_byte_match first_match = rdo_search_first_match(search, data, mask, search_start, + continuation_source, source_hashes, + hashes, incompatible); + + rdo_byte_match second_match = rdo_search_second_match(search, data, search_start, + first_match, incompatible); + + // Preserve exact accepted bytes and track matches that can continue into the next block + continuation_source = -1; + if (search.best_cost < original_cost) + { + std::memcpy(original, search.best_block, block_size); + rdo_byte_match last_match = first_match; + if ((second_match.length != 0) && (second_match.offset > first_match.offset)) + { + last_match = second_match; + } + + if (last_match.offset + last_match.length == block_size) + { + continuation_source = static_cast(last_match.source + last_match.length); + } + } + } +} + +#endif diff --git a/Source/astcenc_vecmathlib_avx2_8.h b/Source/astcenc_vecmathlib_avx2_8.h index 353f99445..4a0bac993 100644 --- a/Source/astcenc_vecmathlib_avx2_8.h +++ b/Source/astcenc_vecmathlib_avx2_8.h @@ -439,10 +439,12 @@ ASTCENC_SIMD_INLINE bool all(vmask8 a) // vint8 operators and functions // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator+(vint8 a, vint8 b) { @@ -450,10 +452,12 @@ ASTCENC_SIMD_INLINE vint8 operator+(vint8 a, vint8 b) } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator-(vint8 a, vint8 b) { @@ -461,10 +465,12 @@ ASTCENC_SIMD_INLINE vint8 operator-(vint8 a, vint8 b) } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator*(vint8 a, vint8 b) { diff --git a/Source/astcenc_vecmathlib_avx512_16.h b/Source/astcenc_vecmathlib_avx512_16.h index a3764bebc..f6cc95e0d 100644 --- a/Source/astcenc_vecmathlib_avx512_16.h +++ b/Source/astcenc_vecmathlib_avx512_16.h @@ -432,10 +432,12 @@ ASTCENC_SIMD_INLINE bool all(vmask16 a) // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint16 operator+(vint16 a, vint16 b) { @@ -443,10 +445,12 @@ ASTCENC_SIMD_INLINE vint16 operator+(vint16 a, vint16 b) } /** - * @brief Overload: vector by vector incremental addition. + * @brief Overload: vector by vector incremental addition, wrapping modulo 2^32. * * @param[in,out] a The accumulator to update. - * @param b The values to add. + * @param[in] b The values to add. + * + * @return The updated accumulator. */ ASTCENC_SIMD_INLINE vint16& operator+=(vint16& a, const vint16& b) { @@ -455,10 +459,12 @@ ASTCENC_SIMD_INLINE vint16& operator+=(vint16& a, const vint16& b) } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint16 operator-(vint16 a, vint16 b) { @@ -466,10 +472,12 @@ ASTCENC_SIMD_INLINE vint16 operator-(vint16 a, vint16 b) } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint16 operator*(vint16 a, vint16 b) { diff --git a/Source/astcenc_vecmathlib_common_4.h b/Source/astcenc_vecmathlib_common_4.h index 2bdd37ee9..c4b313694 100644 --- a/Source/astcenc_vecmathlib_common_4.h +++ b/Source/astcenc_vecmathlib_common_4.h @@ -30,6 +30,7 @@ #error "Include astcenc_vecmathlib.h, do not include directly" #endif +#include #include #include #include @@ -67,10 +68,12 @@ ASTCENC_SIMD_INLINE bool all(vmask4 a) // ============================================================================ /** - * @brief Overload: vector by scalar addition. + * @brief Overload: vector by scalar addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, int b) { @@ -78,10 +81,12 @@ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, int b) } /** - * @brief Overload: vector by vector incremental addition. + * @brief Overload: vector by vector incremental addition, wrapping modulo 2^32. * * @param[in,out] a The accumulator to update. - * @param b The values to add. + * @param[in] b The values to add. + * + * @return The updated accumulator. */ ASTCENC_SIMD_INLINE vint4& operator+=(vint4& a, const vint4& b) { @@ -90,10 +95,12 @@ ASTCENC_SIMD_INLINE vint4& operator+=(vint4& a, const vint4& b) } /** - * @brief Overload: vector by scalar subtraction. + * @brief Overload: vector by scalar subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, int b) { @@ -101,10 +108,12 @@ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, int b) } /** - * @brief Overload: vector by scalar multiplication. + * @brief Overload: vector by scalar multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, int b) { @@ -157,13 +166,18 @@ ASTCENC_SIMD_INLINE vint4 clamp(int minv, int maxv, vint4 a) } /** - * @brief Return the horizontal sum of RGB vector lanes as a scalar. + * @brief Return the horizontal sum of RGB vector lanes as a scalar, wrapping modulo 2^32. * * @param a The vector to reduce. + * + * @return The sum of the RGB lanes, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE int hadd_rgb_s(vint4 a) { - return a.lane<0>() + a.lane<1>() + a.lane<2>(); + uint32_t sum = static_cast(a.lane<0>()) + + static_cast(a.lane<1>()) + + static_cast(a.lane<2>()); + return static_cast(sum); } /** @@ -558,6 +572,252 @@ ASTCENC_SIMD_INLINE void store_lanes_masked(uint8_t* base, vint4 data, vmask4 ma #endif +// ============================================================================ +// Packed byte and bit operations +// ============================================================================ + +/** + * @brief Load a little-endian word from an unaligned byte sequence. + * + * @param data The eight input bytes. + * + * @return The decoded word. + */ +ASTCENC_SIMD_INLINE uint64_t load_le64(const uint8_t* data) +{ + if constexpr (std::endian::native == std::endian::little) + { + uint64_t value; + std::memcpy(&value, data, 8); + return value; + } + else + { + uint64_t value = 0; + for (unsigned int i = 0; i < 8; i++) + { + value |= static_cast(data[i]) << (8 * i); + } + + return value; + } +} + +/** + * @brief Store a prefix of a little-endian word to unaligned memory. + * + * @param value The source word. + * @param[out] output The destination bytes. + * @param count The number of bytes to write, in [0, 8]. + */ +ASTCENC_SIMD_INLINE void store_le64_prefix(uint64_t value, uint8_t* output, unsigned int count) +{ + if constexpr (std::endian::native == std::endian::little) + { + std::memcpy(output, &value, count); + } + else + { + for (unsigned int i = 0; i < count; i++) + { + output[i] = static_cast(value >> (8 * i)); + } + } +} + +#if !defined(ASTCENC_USE_NATIVE_BIT_PERMUTE) + +/** @brief True if arbitrary bit extraction and deposition have native instructions. */ +static constexpr bool is_bit_permute_fast = false; + +/** + * @brief Keep the lowest bits of a word. + * + * @param value The source word. + * @param count The number of low bits to keep, in [0, 64]. + * + * @return The masked word. + */ +ASTCENC_SIMD_INLINE uint64_t mask_low_bits(uint64_t value, unsigned int count) +{ + if (count == 64) + { + return value; + } + + return value & ((1ull << count) - 1); +} + +/** + * @brief Extract a contiguous bit range from a word. + * + * @param value The source word. + * @param offset The first bit, in [0, 63]. + * @param count The number of bits, in [0, 64 - offset]. + * + * @return The extracted bits, starting at bit zero. + */ +ASTCENC_SIMD_INLINE uint64_t extract_bit_range(uint64_t value, unsigned int offset, unsigned int count) +{ + return mask_low_bits(value >> offset, count); +} + +/** + * @brief Compress selected bits into consecutive low bits, preserving their order. + * + * @param value The source word. + * @param mask The source bit positions to select. + * + * @return The selected bits, starting at bit zero. + */ +ASTCENC_SIMD_INLINE uint64_t compress_bits(uint64_t value, uint64_t mask) +{ + uint64_t result = 0; + uint64_t output_bit = 1; + while (mask != 0) + { + uint64_t input_bit = mask & (0ull - mask); + if ((value & input_bit) != 0) + { + result |= output_bit; + } + + mask &= mask - 1; + output_bit <<= 1; + } + + return result; +} + +/** + * @brief Deposit consecutive low bits into selected positions, preserving their order. + * + * @param value The source word. + * @param mask The destination bit positions to fill. + * + * @return The deposited bits, with all other positions zero. + */ +ASTCENC_SIMD_INLINE uint64_t deposit_bits(uint64_t value, uint64_t mask) +{ + uint64_t result = 0; + while (mask != 0) + { + uint64_t output_bit = mask & (0ull - mask); + if ((value & 1) != 0) + { + result |= output_bit; + } + + mask &= mask - 1; + value >>= 1; + } + + return result; +} +#endif + +#if !defined(ASTCENC_USE_NATIVE_UNPACK_BYTE_FIELDS) +/** + * @brief Expand eight consecutive fixed-width fields into packed bytes. + * + * @param packed The packed fields, starting at bit zero. + * @param bits The field width, in [1, 8]. + * + * @return The eight byte values in the low eight bytes, with the high bytes zero. + */ +ASTCENC_SIMD_INLINE vint4 unpack_byte_fields(uint64_t packed, unsigned int bits) +{ + uint8_t values[16] {}; + for (unsigned int i = 0; i < 8; i++) + { + values[i] = static_cast((packed >> (i * bits)) & ((1u << bits) - 1)); + } + + return vint4::load(values); +} +#endif + +#if !defined(ASTCENC_USE_NATIVE_VTABLE_LOOKUP_8BIT) +/** + * @brief Look up sixteen packed byte indices in a 16-entry byte table. + * + * @param table The prepared lookup table. + * @param indices The sixteen byte indices, each in [0, 15]. + * + * @return The sixteen table values as packed bytes. + */ +ASTCENC_SIMD_INLINE vint4 vtable_lookup_8bit(const vtable4_16x8& table, vint4 indices) +{ + uint8_t input[16]; + uint8_t output[16]; + store(indices, input); + for (unsigned int i = 0; i < 16; i += 4) + { + vint4 values = vtable_lookup_32bit(table, vint4(input + i)); + pack_and_store_low_bytes(values, output + i); + } + + return vint4::load(output); +} + +/** + * @brief Look up sixteen packed byte indices in a 32-entry byte table. + * + * @param table The prepared lookup table. + * @param indices The sixteen byte indices, each in [0, 31]. + * + * @return The sixteen table values as packed bytes. + */ +ASTCENC_SIMD_INLINE vint4 vtable_lookup_8bit(const vtable4_32x8& table, vint4 indices) +{ + uint8_t input[16]; + uint8_t output[16]; + store(indices, input); + for (unsigned int i = 0; i < 16; i += 4) + { + vint4 values = vtable_lookup_32bit(table, vint4(input + i)); + pack_and_store_low_bytes(values, output + i); + } + + return vint4::load(output); +} +#endif + +#if !defined(ASTCENC_USE_NATIVE_STORE_DEINTERLEAVED_BYTES) +/** + * @brief Store alternating bytes into two eight-byte output sequences. + * + * @param data The sixteen interleaved input bytes. + * @param[out] even The eight even-indexed bytes. + * @param[out] odd The eight odd-indexed bytes. + */ +ASTCENC_SIMD_INLINE void store_deinterleaved_bytes(vint4 data, uint8_t* even, uint8_t* odd) +{ + uint8_t bytes[16]; + store(data, bytes); + for (unsigned int i = 0; i < 8; i++) + { + even[i] = bytes[2 * i]; + odd[i] = bytes[2 * i + 1]; + } +} +#endif + +#if !defined(ASTCENC_USE_NATIVE_STORE_LOW_8BYTES) +/** + * @brief Store the lowest eight packed bytes to an unaligned address. + * + * @param data The packed bytes. + * @param[out] output The eight-byte destination. + */ +ASTCENC_SIMD_INLINE void store_low_8bytes(vint4 data, uint8_t* output) +{ + uint8_t bytes[16]; + store(data, bytes); + std::memcpy(output, bytes, 8); +} +#endif + /** * @brief Debug function to print a vector of ints. * diff --git a/Source/astcenc_vecmathlib_common_8.h b/Source/astcenc_vecmathlib_common_8.h index 297c54953..ec4836204 100644 --- a/Source/astcenc_vecmathlib_common_8.h +++ b/Source/astcenc_vecmathlib_common_8.h @@ -39,10 +39,12 @@ // ============================================================================ /** - * @brief Overload: vector by vector incremental addition. + * @brief Overload: vector by vector incremental addition, wrapping modulo 2^32. * * @param[in,out] a The accumulator to update. - * @param b The values to add. + * @param[in] b The values to add. + * + * @return The updated accumulator. */ ASTCENC_SIMD_INLINE vint8& operator+=(vint8& a, const vint8& b) { diff --git a/Source/astcenc_vecmathlib_neon_4.h b/Source/astcenc_vecmathlib_neon_4.h index 100f8dfcc..8c0c6f4bd 100644 --- a/Source/astcenc_vecmathlib_neon_4.h +++ b/Source/astcenc_vecmathlib_neon_4.h @@ -531,36 +531,45 @@ ASTCENC_SIMD_INLINE bool all(vmask4 a) // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b) { - return vint4(vaddq_s32(a.m, b.m)); + return vint4(vreinterpretq_s32_u32(vaddq_u32(vreinterpretq_u32_s32(a.m), + vreinterpretq_u32_s32(b.m)))); } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b) { - return vint4(vsubq_s32(a.m, b.m)); + return vint4(vreinterpretq_s32_u32(vsubq_u32(vreinterpretq_u32_s32(a.m), + vreinterpretq_u32_s32(b.m)))); } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b) { - return vint4(vmulq_s32(a.m, b.m)); + return vint4(vreinterpretq_s32_u32(vmulq_u32(vreinterpretq_u32_s32(a.m), + vreinterpretq_u32_s32(b.m)))); } /** diff --git a/Source/astcenc_vecmathlib_none_4.h b/Source/astcenc_vecmathlib_none_4.h index eae81291f..37b145092 100644 --- a/Source/astcenc_vecmathlib_none_4.h +++ b/Source/astcenc_vecmathlib_none_4.h @@ -522,45 +522,51 @@ ASTCENC_SIMD_INLINE bool all(vmask4 a) // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b) { - return vint4(a.m[0] + b.m[0], - a.m[1] + b.m[1], - a.m[2] + b.m[2], - a.m[3] + b.m[3]); + return vint4(static_cast(static_cast(a.m[0]) + static_cast(b.m[0])), + static_cast(static_cast(a.m[1]) + static_cast(b.m[1])), + static_cast(static_cast(a.m[2]) + static_cast(b.m[2])), + static_cast(static_cast(a.m[3]) + static_cast(b.m[3]))); } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b) { - return vint4(a.m[0] - b.m[0], - a.m[1] - b.m[1], - a.m[2] - b.m[2], - a.m[3] - b.m[3]); + return vint4(static_cast(static_cast(a.m[0]) - static_cast(b.m[0])), + static_cast(static_cast(a.m[1]) - static_cast(b.m[1])), + static_cast(static_cast(a.m[2]) - static_cast(b.m[2])), + static_cast(static_cast(a.m[3]) - static_cast(b.m[3]))); } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b) { - return vint4(a.m[0] * b.m[0], - a.m[1] * b.m[1], - a.m[2] * b.m[2], - a.m[3] * b.m[3]); + return vint4(static_cast(static_cast(a.m[0]) * static_cast(b.m[0])), + static_cast(static_cast(a.m[1]) * static_cast(b.m[1])), + static_cast(static_cast(a.m[2]) * static_cast(b.m[2])), + static_cast(static_cast(a.m[3]) * static_cast(b.m[3]))); } /** diff --git a/Source/astcenc_vecmathlib_rvv_n.h b/Source/astcenc_vecmathlib_rvv_n.h index de0f56022..8eefc730a 100644 --- a/Source/astcenc_vecmathlib_rvv_n.h +++ b/Source/astcenc_vecmathlib_rvv_n.h @@ -394,10 +394,12 @@ ASTCENC_SIMD_INLINE bool all(vmask a) // vint operators and functions // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint operator+(vint a, vint b) { @@ -405,10 +407,12 @@ ASTCENC_SIMD_INLINE vint operator+(vint a, vint b) } /** - * @brief Overload: vector by vector incremental addition. + * @brief Overload: vector by vector incremental addition, wrapping modulo 2^32. * * @param[in,out] a The accumulator to update. - * @param b The values to add. + * @param[in] b The values to add. + * + * @return The updated accumulator. */ ASTCENC_SIMD_INLINE vint& operator+=(vint& a, const vint& b) { @@ -417,10 +421,12 @@ ASTCENC_SIMD_INLINE vint& operator+=(vint& a, const vint& b) } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint operator-(vint a, vint b) { @@ -428,10 +434,12 @@ ASTCENC_SIMD_INLINE vint operator-(vint a, vint b) } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint operator*(vint a, vint b) { diff --git a/Source/astcenc_vecmathlib_sse_4.h b/Source/astcenc_vecmathlib_sse_4.h index a657845b1..0b6992304 100644 --- a/Source/astcenc_vecmathlib_sse_4.h +++ b/Source/astcenc_vecmathlib_sse_4.h @@ -523,10 +523,12 @@ ASTCENC_SIMD_INLINE unsigned int mask(vmask4 a) // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b) { @@ -534,10 +536,12 @@ ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b) } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b) { @@ -545,10 +549,12 @@ ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b) } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b) { @@ -1443,6 +1449,149 @@ ASTCENC_SIMD_INLINE vint4 vtable_lookup_32bit( #endif } +// ============================================================================ +// Packed byte and bit operations +// ============================================================================ + +#if ASTCENC_AVX >= 2 +#define ASTCENC_USE_NATIVE_BIT_PERMUTE 1 + +/** @brief True if arbitrary bit extraction and deposition have native instructions. */ +static constexpr bool is_bit_permute_fast = true; + +/** + * @brief Keep the lowest bits of a word. + * + * @param value The source word. + * @param count The number of low bits to keep, in [0, 64]. + * + * @return The masked word. + */ +ASTCENC_SIMD_INLINE uint64_t mask_low_bits(uint64_t value, unsigned int count) +{ + return _bzhi_u64(value, count); +} + +/** + * @brief Extract a contiguous bit range from a word. + * + * @param value The source word. + * @param offset The first bit, in [0, 63]. + * @param count The number of bits, in [0, 64 - offset]. + * + * @return The extracted bits, starting at bit zero. + */ +ASTCENC_SIMD_INLINE uint64_t extract_bit_range(uint64_t value, unsigned int offset, unsigned int count) +{ + return _bextr_u64(value, offset, count); +} + +/** + * @brief Compress selected bits into consecutive low bits, preserving their order. + * + * @param value The source word. + * @param mask The source bit positions to select. + * + * @return The selected bits, starting at bit zero. + */ +ASTCENC_SIMD_INLINE uint64_t compress_bits(uint64_t value, uint64_t mask) +{ + return _pext_u64(value, mask); +} + +/** + * @brief Deposit consecutive low bits into selected positions, preserving their order. + * + * @param value The source word. + * @param mask The destination bit positions to fill. + * + * @return The deposited bits, with all other positions zero. + */ +ASTCENC_SIMD_INLINE uint64_t deposit_bits(uint64_t value, uint64_t mask) +{ + return _pdep_u64(value, mask); +} + +#define ASTCENC_USE_NATIVE_UNPACK_BYTE_FIELDS 1 + +/** + * @brief Expand eight consecutive fixed-width fields into packed bytes. + * + * @param packed The packed fields, starting at bit zero. + * @param bits The field width, in [1, 8]. + * + * @return The eight byte values in the low eight bytes, with the high bytes zero. + */ +ASTCENC_SIMD_INLINE vint4 unpack_byte_fields(uint64_t packed, unsigned int bits) +{ + uint64_t lane_mask = 0x0101010101010101ull * mask_low_bits(~0ull, bits); + return vint4(_mm_cvtsi64_si128(deposit_bits(packed, lane_mask))); +} +#endif + +#if ASTCENC_SSE >= 41 +#define ASTCENC_USE_NATIVE_VTABLE_LOOKUP_8BIT 1 + +/** + * @brief Look up sixteen packed byte indices in a 16-entry byte table. + * + * @param table The prepared lookup table. + * @param indices The sixteen byte indices, each in [0, 15]. + * + * @return The sixteen table values as packed bytes. + */ +ASTCENC_SIMD_INLINE vint4 vtable_lookup_8bit(const vtable4_16x8& table, vint4 indices) +{ + return vint4(_mm_shuffle_epi8(table.t0.m, indices.m)); +} + +/** + * @brief Look up sixteen packed byte indices in a 32-entry byte table. + * + * @param table The prepared lookup table. + * @param indices The sixteen byte indices, each in [0, 31]. + * + * @return The sixteen table values as packed bytes. + */ +ASTCENC_SIMD_INLINE vint4 vtable_lookup_8bit(const vtable4_32x8& table, vint4 indices) +{ + __m128i low = _mm_shuffle_epi8(table.t0.m, indices.m); + __m128i high_indices = _mm_sub_epi8(indices.m, _mm_set1_epi8(16)); + __m128i high = _mm_shuffle_epi8(table.t1.m, high_indices); + return vint4(_mm_xor_si128(low, high)); +} + +#define ASTCENC_USE_NATIVE_STORE_DEINTERLEAVED_BYTES 1 + +/** + * @brief Store alternating bytes into two eight-byte output sequences. + * + * @param data The sixteen interleaved input bytes. + * @param[out] even The eight even-indexed bytes. + * @param[out] odd The eight odd-indexed bytes. + */ +ASTCENC_SIMD_INLINE void store_deinterleaved_bytes(vint4 data, uint8_t* even, uint8_t* odd) +{ + __m128i order = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); + __m128i result = _mm_shuffle_epi8(data.m, order); + _mm_storel_epi64(reinterpret_cast<__m128i*>(even), result); + _mm_storel_epi64(reinterpret_cast<__m128i*>(odd), _mm_srli_si128(result, 8)); +} +#endif + +#define ASTCENC_USE_NATIVE_STORE_LOW_8BYTES 1 + +/** + * @brief Store the lowest eight packed bytes to an unaligned address. + * + * @param data The packed bytes. + * @param[out] output The eight-byte destination. + */ +ASTCENC_SIMD_INLINE void store_low_8bytes(vint4 data, uint8_t* output) +{ + _mm_storel_epi64(reinterpret_cast<__m128i*>(output), data.m); +} + /** * @brief Return a vector of interleaved RGBA data. * diff --git a/Source/astcenc_vecmathlib_sve_8.h b/Source/astcenc_vecmathlib_sve_8.h index fba93f82b..e65e1ccc2 100644 --- a/Source/astcenc_vecmathlib_sve_8.h +++ b/Source/astcenc_vecmathlib_sve_8.h @@ -428,10 +428,12 @@ ASTCENC_SIMD_INLINE bool all(vmask8 a) // vint8 operators and functions // ============================================================================ /** - * @brief Overload: vector by vector addition. + * @brief Overload: vector by vector addition, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator+(vint8 a, vint8 b) { @@ -439,10 +441,12 @@ ASTCENC_SIMD_INLINE vint8 operator+(vint8 a, vint8 b) } /** - * @brief Overload: vector by vector subtraction. + * @brief Overload: vector by vector subtraction, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator-(vint8 a, vint8 b) { @@ -450,10 +454,12 @@ ASTCENC_SIMD_INLINE vint8 operator-(vint8 a, vint8 b) } /** - * @brief Overload: vector by vector multiplication. + * @brief Overload: vector by vector multiplication, wrapping modulo 2^32. * * @param a The left operand. * @param b The right operand. + * + * @return The per-lane result, wrapping modulo 2^32. */ ASTCENC_SIMD_INLINE vint8 operator*(vint8 a, vint8 b) { diff --git a/Source/astcenccli_astcm.cpp b/Source/astcenccli_astcm.cpp new file mode 100644 index 000000000..ca6deec42 --- /dev/null +++ b/Source/astcenccli_astcm.cpp @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @brief Load and store Morton-ordered ASTC blocks without changing the codec's raster layout. + */ + +#include +#include +#include +#include + +#include "astcenccli_internal.h" + +/** @brief The ASTCM header, using the ASTC header layout with a distinct magic. */ +struct astcm_header +{ + /** @brief The ASCII format identifier "ASCM". */ + uint8_t magic[4]; + + /** @brief The block width in texels. */ + uint8_t block_x; + + /** @brief The block height in texels. */ + uint8_t block_y; + + /** @brief The block depth, which must be one. */ + uint8_t block_z; + + /** @brief The image width as a little-endian 24-bit integer. */ + uint8_t dim_x[3]; + + /** @brief The image height as a little-endian 24-bit integer. */ + uint8_t dim_y[3]; + + /** @brief The image depth, which must be one. */ + uint8_t dim_z[3]; +}; + +static_assert(sizeof(astcm_header) == 16); + +/** @brief The ASTCM file identifier. */ +static constexpr uint8_t astcm_magic[4] { 'A', 'S', 'C', 'M' }; + +/** + * @brief Decode a little-endian 24-bit integer. + * + * @param bytes The three encoded bytes. + * + * @return The decoded value. + */ +static size_t unpack_u24( + const uint8_t bytes[3] +) { + return static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16); +} + +/** + * @brief Validate the image layout and compute its block counts and payload size. + * + * @param img The image metadata to validate. + * @param[out] blocks_x The number of blocks across the image. + * @param[out] blocks_y The number of blocks down the image. + * @param[out] data_size The payload size in bytes. + * + * @return @c true on error, @c false on success. + */ +static bool get_astcm_layout( + const astc_compressed_image& img, + size_t& blocks_x, + size_t& blocks_y, + size_t& data_size +) { + if ((img.block_z != 1) || (img.dim_z != 1)) + { + print_error("ERROR: ASTCM only supports 2D images and blocks\n"); + return true; + } + + if ((img.block_x == 0) || (img.block_x > 255) || + (img.block_y == 0) || (img.block_y > 255) || + (img.dim_x == 0) || (img.dim_x > 0xFFFFFF) || + (img.dim_y == 0) || (img.dim_y > 0xFFFFFF)) + { + print_error("ERROR: Invalid ASTCM image or block dimensions\n"); + return true; + } + + blocks_x = astc::get_block_count_safe(img.dim_x, img.block_x); + blocks_y = astc::get_block_count_safe(img.dim_y, img.block_y); + bool is_overflow = false; + data_size = astc::mul_safe(blocks_x, blocks_y, is_overflow); + data_size = astc::mul_safe(data_size, 16, is_overflow); + if (is_overflow || (data_size > img.data.max_size())) + { + print_error("ERROR: ASTCM image size overflow\n"); + return true; + } + + return false; +} + +/** + * @brief Find the raster byte offsets of a brick's blocks in Morton order. + * + * @param blocks_x The number of blocks across the image. + * @param blocks_y The number of blocks down the image. + * @param brick_x The brick origin in blocks. + * @param brick_y The brick origin in blocks. + * @param[out] offsets The byte offsets of in-range blocks in the raster payload. + * + * @return The number of in-range blocks in this brick. + */ +static size_t get_brick_offsets( + size_t blocks_x, + size_t blocks_y, + size_t brick_x, + size_t brick_y, + size_t offsets[64] +) { + size_t count = 0; + for (size_t morton = 0; morton < 64; morton++) + { + // Even Morton bits encode X; odd bits encode Y + size_t x = brick_x + ((morton & 1) | ((morton >> 1) & 2) | ((morton >> 2) & 4)); + size_t y = brick_y + (((morton >> 1) & 1) | ((morton >> 2) & 2) | ((morton >> 3) & 4)); + if ((x < blocks_x) && (y < blocks_y)) + { + offsets[count++] = (y * blocks_x + x) * 16; + } + } + + return count; +} + +/* See header for documentation. */ +bool load_astcm_compressed_image( + const char* filename, + astc_compressed_image& img +) { + std::ifstream file(filename, std::ios::binary); + if (!file) + { + print_error("ERROR: File open failed '%s'\n", filename); + return true; + } + + astcm_header header; + file.read(reinterpret_cast(&header), sizeof(header)); + if (!file || (std::memcmp(header.magic, astcm_magic, sizeof(astcm_magic)) != 0)) + { + print_error("ERROR: Invalid ASTCM header '%s'\n", filename); + return true; + } + + astc_compressed_image loaded { + .block_x = header.block_x, + .block_y = header.block_y, + .block_z = header.block_z, + .dim_x = unpack_u24(header.dim_x), + .dim_y = unpack_u24(header.dim_y), + .dim_z = unpack_u24(header.dim_z), + .data = {} + }; + + size_t blocks_x; + size_t blocks_y; + size_t data_size; + if (get_astcm_layout(loaded, blocks_x, blocks_y, data_size)) + { + return true; + } + + // Check the file length before allocating memory based on untrusted dimensions + file.seekg(0, std::ios::end); + std::streamoff file_size = file.tellg(); + std::streamoff header_size = sizeof(header); + if ((file_size < header_size) || (static_cast(file_size - header_size) != data_size)) + { + print_error("ERROR: ASTCM payload size mismatch '%s'\n", filename); + return true; + } + + file.seekg(sizeof(header), std::ios::beg); + try + { + loaded.data.resize(data_size); + } + catch (const std::bad_alloc&) + { + print_error("ERROR: ASTCM image allocation failed\n"); + return true; + } + + // Transfer one brick at a time to avoid a second image-sized buffer + uint8_t brick[64 * 16]; + size_t offsets[64]; + for (size_t y = 0; y < blocks_y; y += 8) + { + for (size_t x = 0; x < blocks_x; x += 8) + { + size_t count = get_brick_offsets(blocks_x, blocks_y, x, y, offsets); + file.read(reinterpret_cast(brick), static_cast(count * 16)); + if (!file) + { + print_error("ERROR: ASTCM payload read failed '%s'\n", filename); + return true; + } + + for (size_t i = 0; i < count; i++) + { + std::memcpy(loaded.data.data() + offsets[i], brick + i * 16, 16); + } + } + } + + img = std::move(loaded); + return false; +} + +/* See header for documentation. */ +bool write_astcm_compressed_image( + const astc_compressed_image& img, + std::ostream& stream +) { + size_t blocks_x; + size_t blocks_y; + size_t data_size; + if (get_astcm_layout(img, blocks_x, blocks_y, data_size)) + { + return true; + } + + if (img.data.size() != data_size) + { + print_error("ERROR: ASTCM payload size does not match image dimensions\n"); + return true; + } + + astcm_header header {}; + std::memcpy(header.magic, astcm_magic, sizeof(astcm_magic)); + header.block_x = static_cast(img.block_x); + header.block_y = static_cast(img.block_y); + header.block_z = 1; + header.dim_z[0] = 1; + for (unsigned int i = 0; i < 3; i++) + { + header.dim_x[i] = static_cast((img.dim_x >> (i * 8)) & 0xFF); + header.dim_y[i] = static_cast((img.dim_y >> (i * 8)) & 0xFF); + } + + stream.write(reinterpret_cast(&header), sizeof(header)); + uint8_t brick[64 * 16]; + size_t offsets[64]; + for (size_t y = 0; y < blocks_y; y += 8) + { + for (size_t x = 0; x < blocks_x; x += 8) + { + size_t count = get_brick_offsets(blocks_x, blocks_y, x, y, offsets); + for (size_t i = 0; i < count; i++) + { + std::memcpy(brick + i * 16, img.data.data() + offsets[i], 16); + } + + stream.write(reinterpret_cast(brick), + static_cast(count * 16)); + if (!stream) + { + return true; + } + } + } + + return stream.fail(); +} + +/* See header for documentation. */ +bool store_astcm_compressed_image( + const astc_compressed_image& img, + const char* filename +) { + // Validate before opening the output to avoid truncating files for unsupported image layouts + size_t blocks_x; + size_t blocks_y; + size_t data_size; + if (get_astcm_layout(img, blocks_x, blocks_y, data_size)) + { + return true; + } + + if (img.data.size() != data_size) + { + print_error("ERROR: ASTCM payload size does not match image dimensions\n"); + return true; + } + + std::ofstream file(filename, std::ios::binary); + if (!file) + { + print_error("ERROR: File open failed '%s'\n", filename); + return true; + } + + bool is_write_error = write_astcm_compressed_image(img, file); + file.close(); + return is_write_error || file.fail(); +} diff --git a/Source/astcenccli_compression_metrics.cpp b/Source/astcenccli_compression_metrics.cpp new file mode 100644 index 000000000..b71408c8b --- /dev/null +++ b/Source/astcenccli_compression_metrics.cpp @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @brief Measure lossless compression of serialized ASTC and Morton-ordered ASTCM images. + */ + +#if defined(ASTCENC_RDO_SIZE) + +#include +#include +#include +#include + +#include + +#include "astcenccli_internal.h" + +/** + * @brief Measure a level 9 raw DEFLATE stream without retaining the compressed output. + * + * @param data The serialized file to compress. + * @param[out] comp_size The compressed size in bytes, excluding ZIP container metadata. + * + * @return @c true on error, @c false on success. + */ +static bool get_deflate_size( + std::string_view data, + size_t& comp_size +) { + z_stream stream {}; + int status = deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS, + 8, Z_DEFAULT_STRATEGY); + if (status != Z_OK) + { + print_error("ERROR: zlib initialization failed: %d\n", status); + return true; + } + + uint8_t output[64 * 1024]; + size_t offset = 0; + comp_size = 0; + do + { + // Chunk input to fit zlib's unsigned int buffer lengths + if (stream.avail_in == 0) + { + size_t count = std::min(data.size() - offset, + static_cast(std::numeric_limits::max())); + stream.next_in = reinterpret_cast(const_cast(data.data() + offset)); + stream.avail_in = static_cast(count); + offset += count; + } + + int flush = Z_NO_FLUSH; + if (offset == data.size()) + { + flush = Z_FINISH; + } + + stream.next_out = output; + stream.avail_out = sizeof(output); + status = deflate(&stream, flush); + comp_size += sizeof(output) - stream.avail_out; + } while (status == Z_OK); + + int end_status = deflateEnd(&stream); + if ((status != Z_STREAM_END) || (end_status != Z_OK)) + { + print_error("ERROR: zlib compression failed: %d\n", status); + return true; + } + + return false; +} + +/* See header for documentation. */ +bool print_compression_metrics( + const astc_compressed_image& img, + bool is_morton +) { + // Use file serializers so headers, block order, and partial tiles match the selected format + std::ostringstream stream(std::ios::out | std::ios::binary); + const char* format = "ASTC"; + bool is_write_error; + if (is_morton) + { + format = "ASTCM"; + is_write_error = write_astcm_compressed_image(img, stream); + } + else + { + is_write_error = write_cimage(img, stream); + } + + if (is_write_error) + { + print_error("ERROR: Failed to serialize %s for compression metrics\n", format); + return true; + } + + std::string_view data = stream.view(); + size_t comp_size; + bool is_compression_error = get_deflate_size(data, comp_size); + if (is_compression_error) + { + return true; + } + + printf("\nStorage metrics\n"); + printf("===============\n\n"); + + const char* file_label = "ASTC file size:"; + const char* deflate_label = "ASTC DEFLATE size:"; + if (is_morton) + { + file_label = "ASTCM file size:"; + deflate_label = "ASTCM DEFLATE size:"; + } + + double data_sized = static_cast(data.size()) / 1024.0; + double comp_sized = static_cast(comp_size) / 1024.0; + double comp_percent = 100.0 * comp_sized / data_sized; + + printf(" %-27s%8.1f KB\n", file_label, data_sized); + printf(" %-27s%8.1f KB (%.2f%%)\n", deflate_label, comp_sized, comp_percent); + printf(" DEFLATE level: %8d\n", Z_BEST_COMPRESSION); + return false; +} + +#endif diff --git a/Source/astcenccli_image_load_store.cpp b/Source/astcenccli_image_load_store.cpp index 75a8d1d91..3c66d34d6 100644 --- a/Source/astcenccli_image_load_store.cpp +++ b/Source/astcenccli_image_load_store.cpp @@ -2803,9 +2803,9 @@ bool load_cimage( } /* See header for documentation. */ -bool store_cimage( +bool write_cimage( const astc_compressed_image& img, - const char* filename + std::ostream& stream ) { if ((img.dim_x > 0xFFFFFF) || (img.dim_y > 0xFFFFFF) || (img.dim_z > 0xFFFFFF)) { @@ -2839,6 +2839,22 @@ bool store_cimage( hdr.dim_z[1] = (dim_z >> 8) & 0xFF; hdr.dim_z[2] = (dim_z >> 16) & 0xFF; + stream.write(reinterpret_cast(&hdr), sizeof(astc_header)); + stream.write(reinterpret_cast(img.data.data()), img.data.size()); + return stream.fail(); +} + +/* See header for documentation. */ +bool store_cimage( + const astc_compressed_image& img, + const char* filename +) { + if ((img.dim_x > 0xFFFFFF) || (img.dim_y > 0xFFFFFF) || (img.dim_z > 0xFFFFFF)) + { + print_error("ERROR: Image exceeds file format size limits\n"); + return true; + } + std::ofstream file(filename, std::ios::out | std::ios::binary); if (!file) { @@ -2846,8 +2862,7 @@ bool store_cimage( return true; } - file.write(reinterpret_cast(&hdr), sizeof(astc_header)); - file.write(reinterpret_cast(img.data.data()), img.data.size()); + bool is_write_error = write_cimage(img, file); file.close(); - return file.fail(); + return is_write_error || file.fail(); } diff --git a/Source/astcenccli_internal.h b/Source/astcenccli_internal.h index 49b0d6b9c..837339c91 100644 --- a/Source/astcenccli_internal.h +++ b/Source/astcenccli_internal.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -269,6 +270,68 @@ bool store_cimage( const astc_compressed_image& img, const char* filename); +/** + * @brief Serialize a raster-ordered compressed buffer as an ASTC image. + * + * @param img The image to serialize. + * @param[out] stream The output stream, including the ASTC header. + * + * @return @c true on error, @c false on success. + */ +bool write_cimage( + const astc_compressed_image& img, + std::ostream& stream); + +/** + * @brief Load a Morton-ordered .astcm image into a raster-ordered compressed buffer. + * + * @param filename The file to load. + * @param[out] img The image to populate with loaded data. + * + * @return @c true on error, @c false on success. + */ +bool load_astcm_compressed_image( + const char* filename, + astc_compressed_image& img); + +/** + * @brief Store a raster-ordered compressed buffer as a Morton-ordered .astcm image. + * + * @param img The image to store; only 2D images and blocks are supported. + * @param filename The file to save. + * + * @return @c true on error, @c false on success. + */ +bool store_astcm_compressed_image( + const astc_compressed_image& img, + const char* filename); + +/** + * @brief Serialize a raster-ordered compressed buffer as a Morton-ordered .astcm image. + * + * @param img The image to serialize; only 2D images and blocks are supported. + * @param[out] stream The output stream, including the ASTCM header. + * + * @return @c true on error, @c false on success. + */ +bool write_astcm_compressed_image( + const astc_compressed_image& img, + std::ostream& stream); + +#if defined(ASTCENC_RDO_SIZE) +/** + * @brief Report the serialized ASTC or ASTCM size before and after level 9 raw DEFLATE compression. + * + * @param img The compressed image to measure. + * @param is_morton Whether to serialize in Morton order instead of scanline order. + * + * @return @c true on error, @c false on success. + */ +bool print_compression_metrics( + const astc_compressed_image& img, + bool is_morton); +#endif + /** * @brief Load a compressed .ktx image. * diff --git a/Source/astcenccli_toplevel.cpp b/Source/astcenccli_toplevel.cpp index 2f8aa1392..a716cca16 100644 --- a/Source/astcenccli_toplevel.cpp +++ b/Source/astcenccli_toplevel.cpp @@ -925,6 +925,38 @@ static bool edit_astcenc_config( argidx++; cli_config.silentmode = 1; } +#if defined(ASTCENC_RDO) + else if (!strcmp(argv[argidx], "-rdo-lambda")) + { + const char* option = argv[argidx]; + bool is_arg_error = advance_option(option, argc, 1, argidx); + if (is_arg_error) + { + return true; + } + + if (!(operation & ASTCENC_STAGE_COMPRESS)) + { + print_error("ERROR: %s switch requires compression mode\n", option); + return true; + } + + float value; + bool is_parse_error = parse_float(option, argv[argidx - 1], value); + if (is_parse_error) + { + return true; + } + + if (value < 0.0f) + { + print_error("ERROR: Value for %s switch must be nonnegative\n", option); + return true; + } + + config.rdo_lambda = value; + } +#endif else if (!strcmp(argv[argidx], "--croporigin")) { const char* option = argv[argidx]; @@ -1664,6 +1696,13 @@ static void print_astcenc_config( printf(" G component weight: %g\n", static_cast(config.cw_g_weight)); printf(" B component weight: %g\n", static_cast(config.cw_b_weight)); printf(" A component weight: %g\n", static_cast(config.cw_a_weight)); +#if defined(ASTCENC_RDO) + if (config.rdo_lambda > 0.0f) + { + printf(" RDO: Enabled\n"); + printf(" RDO lambda: %g\n", static_cast(config.rdo_lambda)); + } +#endif printf(" Partition cutoff: %u partitions\n", config.tune_partition_count_limit); printf(" 2 partition index cutoff: %u partition ids\n", config.tune_2partition_index_limit); printf(" 3 partition index cutoff: %u partition ids\n", config.tune_3partition_index_limit); @@ -2515,6 +2554,14 @@ int astcenc_main( return 1; } } + else if (ends_with(input_filename, ".astcm")) + { + error = load_astcm_compressed_image(input_filename.c_str(), image_comp); + if (error) + { + return 1; + } + } else if (ends_with(input_filename, ".ktx")) { bool is_srgb; @@ -2620,7 +2667,8 @@ int astcenc_main( if (operation & ASTCENC_STAGE_ST_COMP) { - if (!(is_null || ends_with(output_filename, ".astc") || ends_with(output_filename, ".ktx"))) + if (!(is_null || ends_with(output_filename, ".astc") || + ends_with(output_filename, ".astcm") || ends_with(output_filename, ".ktx"))) { const char *eptr = strrchr(output_filename.c_str(), '.'); eptr = eptr ? eptr : ""; @@ -2779,6 +2827,12 @@ int astcenc_main( } } + // Match encoder traversal and RDO history to the compressed output layout + if ((operation & ASTCENC_STAGE_ST_COMP) && ends_with(output_filename, ".astcm")) + { + config.flags |= ASTCENC_FLG_USE_MORTON; + } + astcenc_context* codec_context_raw { nullptr }; codec_status = astcenc_context_alloc(&config, cli_config.thread_count, &codec_context_raw, nullptr); if (codec_status != ASTCENC_SUCCESS) @@ -2968,6 +3022,15 @@ int astcenc_main( return 1; } } + else if (ends_with(output_filename, ".astcm")) + { + error = store_astcm_compressed_image(image_comp, output_filename.c_str()); + if (error) + { + print_error("ERROR: Failed to store compressed image\n"); + return 1; + } + } else if (ends_with(output_filename, ".ktx")) { bool srgb = profile == ASTCENC_PRF_LDR_SRGB; @@ -3043,6 +3106,20 @@ int astcenc_main( printf(" Decoding time: %8.4f s\n", decompression_time); printf(" Decoding rate: %8.4f MT/s\n", decompression_rate); } +#if defined(ASTCENC_RDO_SIZE) + // Always print size metrics + bool is_morton = config.flags & ASTCENC_FLG_USE_MORTON; + if (!(operation & ASTCENC_STAGE_COMPRESS)) + { + is_morton = ends_with(input_filename, ".astcm"); + } + + bool is_metrics_error = print_compression_metrics(image_comp, is_morton); + if (is_metrics_error) + { + return 1; + } +#endif } if (is_decompression_warning) diff --git a/Source/astcenccli_toplevel_help.cpp b/Source/astcenccli_toplevel_help.cpp index 06e8d776e..e8f3d0cd2 100644 --- a/Source/astcenccli_toplevel_help.cpp +++ b/Source/astcenccli_toplevel_help.cpp @@ -554,13 +554,19 @@ COMPRESSION FILE FORMATS The following formats are supported as compression outputs: * ASTC (*.astc) + * Morton-ordered ASTC (*.astcm, 2D only) * Khronos Texture KTX (*.ktx) + Compression uses 16x8 Morton-ordered encoder bricks for ASTCM output, + and 128x1 scanline-ordered encoder bricks for linear ASTC and KTX output. + These dimensions are in ASTC blocks. ASTCM file tiles remain 8x8 blocks. + DECOMPRESSION FILE FORMATS The following formats are supported as decompression inputs: * ASTC (*.astc) + * Morton-ordered ASTC (*.astcm, 2D only) * Khronos Texture KTX (*.ktx) The following formats are supported as decompression outputs: diff --git a/Source/cmake_core.cmake b/Source/cmake_core.cmake index f4c4a7c47..cc8097ef6 100644 --- a/Source/cmake_core.cmake +++ b/Source/cmake_core.cmake @@ -21,6 +21,16 @@ project(${ASTCENC_TARGET}) include(cmake_compiler.cmake) +set(ASTCENC_RDO_SOURCES "") +if(ASTCENC_RDO AND (NOT ASTCENC_DECOMPRESSOR)) + set(ASTCENC_RDO_SOURCES + astcenc_rdo_brick.cpp + astcenc_rdo_decode.cpp + astcenc_rdo_rate.cpp + astcenc_rdo_scoring.cpp + astcenc_rdo_search.cpp) +endif() + add_library(${ASTCENC_TARGET}-static STATIC astcenc_averages_and_directions.cpp @@ -42,6 +52,7 @@ add_library(${ASTCENC_TARGET}-static astcenc_percentile_tables.cpp astcenc_pick_best_endpoint_format.cpp astcenc_quantization.cpp + ${ASTCENC_RDO_SOURCES} astcenc_symbolic_physical.cpp astcenc_weight_align.cpp astcenc_weight_quant_xfer_tables.cpp) @@ -73,6 +84,7 @@ if(${ASTCENC_SHAREDLIB}) astcenc_percentile_tables.cpp astcenc_pick_best_endpoint_format.cpp astcenc_quantization.cpp + ${ASTCENC_RDO_SOURCES} astcenc_symbolic_physical.cpp astcenc_weight_align.cpp astcenc_weight_quant_xfer_tables.cpp) @@ -95,6 +107,7 @@ if(${ASTCENC_CLI}) astcenccli_entry2.cpp) add_executable(${ASTCENC_TARGET} + astcenccli_astcm.cpp astcenccli_error_metrics.cpp astcenccli_image.cpp astcenccli_image_external.cpp @@ -112,6 +125,12 @@ if(${ASTCENC_CLI}) ${ASTCENC_TARGET}-veneer1 ${ASTCENC_TARGET}-veneer2 ${ASTCENC_TARGET}-static) + + if(ASTCENC_RDO_SIZE) + target_sources(${ASTCENC_TARGET} PRIVATE astcenccli_compression_metrics.cpp) + target_compile_definitions(${ASTCENC_TARGET} PRIVATE ASTCENC_RDO_SIZE) + target_link_libraries(${ASTCENC_TARGET} PRIVATE ZLIB::ZLIB) + endif() endif() macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) @@ -126,6 +145,12 @@ macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) ASTCENC_DECOMPRESS_ONLY) endif() + if(ASTCENC_RDO_SOURCES) + target_compile_definitions(${ASTCENC_TARGET_NAME} + PUBLIC + ASTCENC_RDO) + endif() + if(${ASTCENC_BLOCK_MAX_TEXELS}) target_compile_definitions(${ASTCENC_TARGET_NAME} PRIVATE @@ -434,7 +459,7 @@ macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) target_compile_options(${ASTCENC_TARGET_NAME} PRIVATE $<$:-mprefer-vector-width=256> - -mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mpopcnt -mf16c + -mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mbmi -mbmi2 -mpopcnt -mf16c $<${is_gnu_fe}:-Wno-unused-command-line-argument>) endif() @@ -472,8 +497,8 @@ macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) target_compile_options(${ASTCENC_TARGET_NAME} PRIVATE $<${is_msvc_fe}:/arch:AVX2> - $<${is_clangcl}:-mavx2 -mpopcnt -mf16c> - $<${is_gnu_fe}:-mavx2 -mpopcnt -mf16c> + $<${is_clangcl}:-mavx2 -mbmi -mbmi2 -mpopcnt -mf16c> + $<${is_gnu_fe}:-mavx2 -mbmi -mbmi2 -mpopcnt -mf16c> $<${is_gnu_fe}:-Wno-unused-command-line-argument>) endif() diff --git a/Test/astc_profile_rdo.py b/Test/astc_profile_rdo.py new file mode 100755 index 000000000..e7d90ded3 --- /dev/null +++ b/Test/astc_profile_rdo.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# ----------------------------------------------------------------------------- +# Copyright 2026 Arm Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# ----------------------------------------------------------------------------- +''' +Compare ASTC file compressibility and image quality for two astcenc binaries. + +Rate distortion optimization (RDO) aims to change the ASTC encoding to make the data file more +compressible when bundled into a distribution format, such as an Android APK or asset bundle. This +script tests the impact of RDO on both data compressibility and image quality. + +A is the baseline and B is the candidate. Both currently receive identical encoding settings; +candidate-only RDO options will be added when the encoder supports them. Each case runs compression +to ASTC and compression/decompression to PNG for both binaries, then compresses the ASTC files +using ZIP and LZ4. All sizes include file headers. + +Profiling timing is disabled by default. With --repeats N, run each ASTC encoding N times and +report its mean compression time (including any RDO pass) in seconds. +''' + +import argparse +import csv +from dataclasses import dataclass +import io +import math +from pathlib import Path +import statistics +import subprocess as sp +import sys +import zipfile + +import lz4.frame + +from testlib.profile import select_quality_metrics + + +BLOCK_SIZES = ('4x4', '5x4', '5x5', '6x5', '6x6', '8x5', '8x6', '10x5', '10x6', '8x8') + +PROFILES = { + 'ldr': 'l', + 'srgb': 's', + 'hdr': 'h', + 'hdra': + 'H' +} + +QUALITIES = ('fastest', 'fast', 'medium', 'thorough', 'verythorough', 'exhaustive') + +COLUMNS = ( + ('image', None, 'Image', 0), + ('block_size', None, 'Block', 0), + ('compressor', None, 'Codec', 0), + ('compressed_b_kb', 'compressed_diff_kb', 'Cmp Sz (KB)', 2), + ('compressed_b_pct', 'compressed_diff_pct', 'Cmp %', 2), + ('time_b', 'time_diff', 'Time (s)', 4), + ('psnr_b', 'psnr_diff', 'PSNR (dB)', 2), + ('ssim_b', 'ssim_diff', 'SSIM', 2), +) + +METADATA = ( + 'source_image', + 'binary_a', + 'binary_b', + 'format_a', + 'format_b', + 'profile', + 'quality', + 'threads', + 'repeats', + 'zip_level', + 'lz4_level' +) + + +@dataclass(frozen=True) +class EncodingResult: + ''' + An ASTC bitstream and the quality reported by its matching test-mode invocation. + + Attributes: + data: The complete ASTC or ASTCM file, including its header. + quality: The PSNR and SSIM scores keyed by their CLI labels. + coding_time: The mean ASTC encoding time in seconds, or None when timing is disabled. + ''' + data: bytes + quality: dict[str, float] + coding_time: float | None = None + + +def parse_quality(output: str) -> dict[str, float]: + ''' + Read PSNR and SSIM scores without depending on reported processing times. + + Args: + output: The CLI test-mode output. + + Return: + The available PSNR and SSIM scores keyed by label. + + Raises: + ValueError: A reported score is invalid. + ''' + quality = {} + labels = ('PSNR (LDR-RGB)', 'PSNR (LDR-RGBA)', 'mPSNR (RGB)', + 'SSIM (LDR-RGB)', 'SSIM (LDR-RGBA)', 'mSSIM (RGB)') + + for line in output.splitlines(): + name, _, value = line.strip().partition(':') + + if name in labels: + if not value.strip(): + raise ValueError(f'Codec reported an empty {name}') + + score = float(value.split()[0]) + + if not math.isfinite(score): + raise ValueError(f'Codec reported a non-finite {name}') + + quality[name] = score + + return quality + + +def parse_coding_time(output: str) -> float: + ''' + Read the CLI encoding time, allowing short runs rounded to zero seconds. + + Args: + output: The CLI compression-mode output. + + Return: + The reported encoding time in seconds. + + Raises: + ValueError: The encoding time is missing, negative, or non-finite. + ''' + for line in output.splitlines(): + name, _, value = line.strip().partition(':') + + if name == 'Coding time': + fields = value.split() + + if len(fields) != 2 or fields[1] != 's': + raise ValueError('Codec reported an invalid Coding time') + + coding_time = float(fields[0]) + + if not math.isfinite(coding_time) or coding_time < 0.0: + raise ValueError('Codec reported an invalid Coding time') + + return coding_time + + raise ValueError('Codec did not report Coding time') + + +def encode_image(args: argparse.Namespace, binary: str, image: Path, + block_size: str, output_path: Path) -> EncodingResult: + ''' + Encode a texture file, optionally measure repeated encodings, and save a PNG and quality report. + + Args: + args: The shared encoding settings. + binary: The codec executable. + image: The source image. + block_size: The ASTC block size. + output_path: The output bitstream path, with an .astc or .astcm extension. + + Return: + The encoded file bytes, reported quality scores, and optional mean encoding time. + + Raises: + CalledProcessError: A codec invocation failed; its output is saved in the matching log. + ''' + output = '' + profile = PROFILES[args.profile] + coding_times = [] + + for operation, target in (('c', output_path), ('t', output_path.with_suffix('.png'))): + repeats = 1 + + if operation == 'c' and args.repeats > 0: + repeats = args.repeats + + for repeat in range(repeats): + command = [binary, f'-{operation}{profile}', str(image), + str(target), block_size, f'-{args.quality}', + '-j', str(args.threads)] + result = sp.run(command, stdout=sp.PIPE, stderr=sp.STDOUT, text=True, check=False) + log_suffix = f'.{operation}.log' + + if operation == 'c' and args.repeats > 0: + log_suffix = f'.c.{repeat + 1}.log' + + output_path.with_suffix(log_suffix).write_text(result.stdout, encoding='utf-8') + result.check_returncode() + output = result.stdout + + if operation == 'c' and args.repeats > 0: + coding_times.append(parse_coding_time(output)) + + coding_time = None + + if coding_times: + coding_time = statistics.mean(coding_times) + + return EncodingResult(data=output_path.read_bytes(), + quality=parse_quality(output), coding_time=coding_time) + + +def compress_astc(data: bytes, zip_level: int, lz4_level: int) -> dict[str, bytes]: + ''' + Compress a complete ASTC or ASTCM file into a deterministic ZIP archive and an LZ4 frame. + + Args: + data: The encoded file bytes. + zip_level: The ZIP Deflate compression level. + lz4_level: The LZ4 compression level. + + Return: + The complete compressed files keyed by their format names. + ''' + buffer = io.BytesIO() + + # A fixed neutral name prevents filenames, extensions, or timestamps from biasing comparisons + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr(zipfile.ZipInfo('texture.bin'), data, + compress_type=zipfile.ZIP_DEFLATED, compresslevel=zip_level) + + frame = lz4.frame.compress(data, compression_level=lz4_level, + block_size=lz4.frame.BLOCKSIZE_MAX64KB, block_linked=True, + content_checksum=False, block_checksum=False, store_size=True) + + return {'ZIP': buffer.getvalue(), 'LZ4': frame} + + +def compare_quality(image: Path, profile: str, result_a: EncodingResult, + result_b: EncodingResult) -> list[dict[str, str | float]]: + ''' + Compare matching PSNR and SSIM scores using the existing image-test channel policy. + + Args: + image: The source image. + profile: The ASTC color profile. + result_a: The baseline encoding result. + result_b: The candidate encoding result. + + Return: + The quality comparison for the selected channel group. + + Raises: + ValueError: The binaries do not both report the required PSNR and SSIM scores. + ''' + metrics = select_quality_metrics([f'-t{PROFILES[profile]}', str(image)], result_a.quality) + + if not metrics: + raise ValueError('Codec did not report PSNR scores') + + rows: list[dict[str, str | float]] = [] + + # Prefer RGBA when a generic filename reports both channel groups + for psnr_label in metrics[-1:]: + ssim_label = psnr_label.replace('PSNR', 'SSIM') + + for side, result in (('A', result_a), ('B', result_b)): + for label in (psnr_label, ssim_label): + if label not in result.quality: + raise ValueError(f'{side} did not report {label}; ' + 'use a binary with PSNR and SSIM support') + + psnr_a = result_a.quality[psnr_label] + psnr_b = result_b.quality[psnr_label] + ssim_a = result_a.quality[ssim_label] + ssim_b = result_b.quality[ssim_label] + + rows.append({ + 'psnr_a': psnr_a, 'psnr_b': psnr_b, 'psnr_diff': psnr_b - psnr_a, + 'ssim_a': ssim_a, 'ssim_b': ssim_b, 'ssim_diff': ssim_b - ssim_a, + }) + + return rows + + +def compare_case(args: argparse.Namespace, image: Path, block_size: str, + directory: Path) -> list[dict[str, str | int | float]]: + ''' + Run both binaries and compare their compressed sizes and matching quality metrics. + + Args: + args: The benchmark settings. + image: The source image. + block_size: The ASTC block size. + directory: The output directory for this case. + + Return: + One report row per lossless compressor and selected channel group. + ''' + directory.mkdir(parents=True, exist_ok=True) + result_a = encode_image(args, args.binary_a, image, block_size, + directory / f'a.{args.format_a}') + result_b = encode_image(args, args.binary_b, image, block_size, + directory / f'b.{args.format_b}') + qualities = compare_quality(image, args.profile, result_a, result_b) + archives_a = compress_astc(result_a.data, args.zip_level, args.lz4_level) + archives_b = compress_astc(result_b.data, args.zip_level, args.lz4_level) + timing = {} + + if args.repeats > 0: + assert result_a.coding_time is not None + assert result_b.coding_time is not None + timing = {'time_a': result_a.coding_time, 'time_b': result_b.coding_time, + 'time_diff': result_b.coding_time - result_a.coding_time} + + rows: list[dict[str, str | int | float]] = [] + + for compressor, data_a in archives_a.items(): + data_b = archives_b[compressor] + (directory / f'a.{args.format_a}.{compressor.lower()}').write_bytes(data_a) + (directory / f'b.{args.format_b}.{compressor.lower()}').write_bytes(data_b) + + for quality in qualities: + rows.append({ + **quality, + **timing, + 'image': image.name, + 'block_size': block_size, + 'compressor': compressor, + 'compressed_a_kb': len(data_a) / 1024.0, + 'compressed_b_kb': len(data_b) / 1024.0, + 'compressed_diff_kb': (len(data_b) - len(data_a)) / 1024.0, + 'compressed_a_pct': 100.0 * len(data_a) / len(result_a.data), + 'compressed_b_pct': 100.0 * len(data_b) / len(result_b.data), + 'compressed_diff_pct': (100.0 * len(data_b) / len(result_b.data) + - 100.0 * len(data_a) / len(result_a.data)), + 'source_image': str(image.resolve()), + 'binary_a': args.binary_a, + 'binary_b': args.binary_b, + 'format_a': args.format_a, + 'format_b': args.format_b, + 'profile': args.profile, + 'quality': args.quality, + 'threads': args.threads, + 'repeats': args.repeats, + 'zip_level': args.zip_level, + 'lz4_level': args.lz4_level, + }) + + return rows + + +def format_comparison(value: float, difference: float, precision: int) -> str: + ''' + Format a B value with its signed difference from A in brackets. + + Args: + value: The candidate B value. + difference: The candidate value minus the baseline A value. + precision: The number of decimal places for both values. + + Return: + The candidate value followed by its signed difference in brackets. + ''' + sign = '+' + + if difference < 0.0: + sign = '-' + + return f'{value:.{precision}f} ({sign} {abs(difference):.{precision}f})' + + +def report_results(rows: list[dict[str, str | int | float]], csv_path: Path | None) -> None: + ''' + Print the comparison table and optionally save the results and settings as CSV. + + Args: + rows: The benchmark results. + csv_path: The optional destination CSV file. + ''' + is_timed = bool(rows and 'time_b' in rows[0]) + columns = [column for column in COLUMNS if is_timed or column[0] != 'time_b'] + table = [[title for _, _, title, _ in columns]] + + for row in rows: + cells = [] + + for key, difference_key, _, precision in columns: + if difference_key is not None: + cell = format_comparison(float(row[key]), float(row[difference_key]), precision) + else: + cell = str(row[key]) + + cells.append(cell) + + table.append(cells) + + widths = [max(len(cell) for cell in column) for column in zip(*table)] + print('Values are B (B - A); 1 KB = 1024 bytes.') + print('Cmp % is compressed / encoded file size x 100; changes are percentage points.') + + if is_timed: + print('Times are mean ASTC encoding seconds from the CLI Coding time report.') + + for cells in table: + print('| ' + ' | '.join(cell.rjust(width) for cell, width in zip(cells, widths)) + ' |') + + if csv_path is not None: + csv_path.parent.mkdir(parents=True, exist_ok=True) + + with csv_path.open('w', encoding='utf-8', newline='') as handle: + fields = [] + + for key, difference_key, _, _ in columns: + fields.append(key) + + if difference_key is not None: + fields.append(difference_key) + + fields.extend(METADATA) + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + + for row in rows: + record = {key: row[key] for key in fields} + + for key, value in record.items(): + if isinstance(value, float) and key.startswith(('psnr_', 'ssim_')): + record[key] = f'{value:.2f}' + + writer.writerow(record) + + +def parse_command_line(argv: list[str] | None = None) -> argparse.Namespace: + ''' + Parse the input images and shared A/B encoding and compression settings. + + Args: + argv: Arguments to parse, or None to use the process command line. + + Return: + The validated command-line settings. + ''' + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument( + 'binary_a', type=argparse.FileType('rb'), + help='baseline astcenc executable') + + parser.add_argument( + 'binary_b', type=argparse.FileType('rb'), + help='candidate astcenc executable; may be identical to A') + + parser.add_argument( + 'images', nargs='+', type=Path, help='input images') + + parser.add_argument( + '--format-a', choices=('astc', 'astcm'), default='astc', + help='baseline bitstream file format (default: %(default)s)') + + parser.add_argument( + '--format-b', choices=('astc', 'astcm'), default='astcm', + help='candidate bitstream file format (default: %(default)s)') + + parser.add_argument( + '--block-size', dest='block_sizes', action='append', choices=BLOCK_SIZES, + help='block size to test (repeatable, default: all < 64 texels)') + + parser.add_argument( + '--quality', choices=QUALITIES, default='medium', + help='encoding quality preset (default: %(default)s)') + + parser.add_argument( + '--profile', choices=PROFILES, default='ldr', + help='compressor color profile (default: %(default)s)') + + parser.add_argument( + '-j', '--threads', type=int, default=12, + help='compressor threads (default: %(default)s)') + + parser.add_argument( + '--repeats', type=int, default=0, + help='test repeats for timing, 0 disables timing (default: %(default)s)') + + parser.add_argument( + '--zip-level', type=int, choices=range(10), default=7, + help='ZIP Deflate compression level (default: %(default)s)') + + parser.add_argument( + '--lz4-level', type=int, choices=range(17), default=9, + help='LZ4 compression level, 0 is fast and 3+ is HC (default: %(default)s)') + + # TODO: Remove this and make a TempDir in future + parser.add_argument( + '--output-dir', type=Path, default=Path('TestOutput/RDO'), + help='directory for bitstreams, PNGs, archives, and logs (default: %(default)s)') + + parser.add_argument( + '--csv', + type=Path, help='optional output CSV with results and settings') + + args = parser.parse_args(argv) + + for name in ('binary_a', 'binary_b'): + # Retain the filename and close the FileType handle when the "with" block exits + with getattr(args, name) as binary: + setattr(args, name, binary.name) + + if args.threads < 1: + parser.error('--threads must be positive') + + if args.repeats < 0: + parser.error('--repeats cannot be negative') + + args.block_sizes = list(dict.fromkeys(args.block_sizes or BLOCK_SIZES)) + + for image in args.images: + if not image.is_file(): + parser.error(f'cannot find input image: {image}') + + return args + + +def main() -> int: + ''' + Run each image and block-size comparison sequentially and report the results. + + Return: + Zero on success, or one if encoding, compression, or reporting fails. + ''' + args = parse_command_line() + rows = [] + + print(f'A: {args.binary_a} (.{args.format_a})\n' + f'B: {args.binary_b} (.{args.format_b})', file=sys.stderr) + print(f'Profile: {args.profile}, quality: {args.quality}, threads: {args.threads}; ' + f'ZIP level: {args.zip_level}, LZ4 level: {args.lz4_level}', file=sys.stderr) + + try: + for index, image in enumerate(args.images): + for block_size in args.block_sizes: + print(f'Encoding {image} at {block_size}', file=sys.stderr, flush=True) + directory = args.output_dir / f'{index:04d}-{image.stem}' / block_size + rows.extend(compare_case(args, image, block_size, directory)) + + report_results(rows, args.csv) + except (OSError, ValueError, RuntimeError, sp.CalledProcessError) as ex: + print(f'ERROR: {ex}', file=sys.stderr) + + if isinstance(ex, sp.CalledProcessError) and ex.output: + print(ex.output, file=sys.stderr) + + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/requirements.txt b/requirements.txt index 6d3226f4d..2b0e4f1e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,9 @@ numpy Pillow scipy +# RDO file compression comparisons +lz4 + # Python static analysis and style checks mypy pycodestyle