Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions .github/workflows/build_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -183,15 +183,15 @@ jobs:
- name: Update apt packages
run: sudo apt-get update

- name: Install ImageMagick
run: sudo apt-get install imagemagick
- name: Install dependencies
run: sudo apt-get install -y imagemagick zlib1g-dev

- name: Build release
run: |
export CXX=clang++
mkdir build_rel
cd build_rel
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_UNITTEST=ON -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX256=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON -DASTCENC_UNITTEST=ON -DASTCENC_PACKAGE=x64 ..
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_UNITTEST=ON -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX256=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_ISA_NONE=ON -DASTCENC_UNITTEST=ON -DASTCENC_PACKAGE=x64 -DASTCENC_RDO_SIZE=ON ..
make install package -j4

- name: Build release decompressor
Expand Down Expand Up @@ -272,6 +272,12 @@ jobs:
echo "Skipping avx256/avx512 execute tests: host does not support the AVX-512 variants"
fi

- name: Run RDO image test
run: |
./bin/astcenc-avx2 -tl Test/Images/Small/LDR-RGB/ldr-rgb-00.png build_rel/rdo.png \
6x6 -medium -rdo-lambda 20 -j 12 | tee build_rel/rdo.log
grep -q "ASTC DEFLATE size:" build_rel/rdo.log

build-ubuntu-x64-gcc:
name: Ubuntu x64 GCC
runs-on: ubuntu-24.04
Expand Down
6 changes: 6 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,16 @@ option(ASTCENC_UBSAN "Enable astcenc builds with undefined behavior sanitizer")
option(ASTCENC_UNITTEST "Enable astcenc builds with unit tests")
option(ASTCENC_INVARIANCE "Enable astcenc floating point invariance" ON)
option(ASTCENC_CLI "Enable build of astcenc command line tools" ON)
option(ASTCENC_RDO_SIZE "Enable CLI DEFLATE size reporting using zlib" OFF)
option(ASTCENC_X86_GATHERS "Enable use of native x86 gathers" ON)
option(ASTCENC_WERROR "Force builds to treat warnings as errors" ON)

set(ASTCENC_CLI_POSTFIX "" CACHE STRING "Custom postfix for command line tool binary names")

if(ASTCENC_CLI AND ASTCENC_RDO_SIZE)
find_package(ZLIB REQUIRED)
endif()

if(ASTCENC_ASAN AND ASTCENC_TSAN)
message(FATAL_ERROR "ASTCENC_ASAN and ASTCENC_TSAN cannot be enabled together.")
endif()
Expand Down Expand Up @@ -167,6 +172,7 @@ endif()
printopt("Shared libs " ${ASTCENC_SHAREDLIB})
printopt("Decompressor " ${ASTCENC_DECOMPRESSOR})
message(STATUS "Developer options")
printopt("DEFLATE reporting " ${ASTCENC_RDO_SIZE})
printopt("Diagnostics " ${ASTCENC_DIAGNOSTICS})
printopt("ASAN " ${ASTCENC_ASAN})
printopt("TSAN " ${ASTCENC_TSAN})
Expand Down
16 changes: 16 additions & 0 deletions Docs/Building.md
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,22 @@ cd build
ctest --verbose
```

### RDO deflate size reporting

Enable `-DASTCENC_RDO_SIZE=ON` to report ASTC payload size and level 9 raw
DEFLATE size in CLI test mode for an LDR image (`-tl`, or `-ts`) when the RDO
lambda is non-zero. This option is disabled by default and requires the system
to have the zlib development library installed when building the CLI.

To link zlib statically, use CMake 3.24 or newer and add
`-DZLIB_USE_STATIC_LIBS=ON` when configuring a new build directory. This
requires the zlib static library to be installed and avoids a shared zlib
dependency.

The measurement compresses only the ASTC block data, excluding file headers
and ZIP container metadata. It runs after the codec timing measurements, and
does not actually change the encoded image or output file format.

### Sanitizer builds

We support building with sanitizers on Linux and macOS when using Clang.
Expand Down
74 changes: 74 additions & 0 deletions Docs/RDO.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Rate-distortion optimization

Rate-distortion optimization (RDO) trades image quality for smaller files after
applying additional lossless compression to compressed ASTC files. It is used
to reduce the size of application distribution bundles, providing finer-grained
control over storage size than simply reducing ASTC block size.

> [!NOTE]
> The RDO implementation in `astcenc` currently only supports LDR textures.

## How RDO works

ASTC compression is a fixed-rate compression that encodes every block into 16
bytes of data. For a given image size and block footprint, the ASTC payload
data has a fixed size. RDO is a post-process that changes the compressed ASTC
bitstream to make byte patterns more similar to nearby bytes in the stream.
This makes the data more compressible when applying subsequent lossless
compression, such as ZIP or LZ4.

The RDO transform is lossy, sacrificing image quality to improve data
compressibility. The amount of allowed quality loss is controlled by a lambda
setting, `-rdo-lambda` for the command line tool, where a higher lambda values
allow more image distortion in exchange for a more compressible pattern. A
lambda value of zero disables the RDO transform. We recommend using a value
of 20 as a starting point, which give a reasonable tradeoff between quality
and storage. Normal maps are more sensitive to introduced error, so a lower
lambda value might be required.

The ASTC blocks after RDO has been applied are still standard ASTC data blocks.
RDO does not require a special GPU decoder, and it does not change the number of
blocks or the size of each block. To realize the storage saving, additional
lossless compression must be applied in your asset packaging pipeline. For
Android, this is compression is commonly applied when generating an Application
APK (application bundle) or AAB (asset bundle).

### Using RDO versus changing the ASTC block size

When using ASTC, you have two controls that can reduce packaged file size. You
can choose to use a larger ASTC block size, which simply reduces the amount of
data that needs to be packaged, or you can choose to use RDO to make data more
compressible.

Where possible we recommend using a larger ASTC block size. This gives you
additional savings to run-time memory bandwidth, in addition to the packging
size reduction.

We recommend using RDO to fine-tune packaging size when dropping to the next
lowest ASTC bitrate is not an option. This could be because the next bitrate
option loses too much detail, or suffers from more visible block artifacts.

## Compressing with RDO

Use the `-rdo-lambda` command line option to enable RDO and set its strength.
Setting 0 disables RDO, higher values increase the amount of quality loss
permitted.

```shell
# Reference encode without RDO
build-rdo/Source/astcenc-avx2 -cl input.png reference.astc 5x5 -thorough \
-rdo-lambda 0

# Same footprint and search preset, with a light RDO pass enabled
build-rdo/Source/astcenc-avx2 -cl input.png rdo.astc 5x5 -thorough \
-rdo-lambda 20
```

The RDO lambda value is a heuristic to control tradeoffs, not a specific target
bitrate or PSNR threshold. Lower positive values favor fidelity; higher values
place more weight on storage savings. You much measure quality and check that
it is suitable for your content.

- - -

_Copyright © 2026, Arm Limited and contributors._
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ It covers:
* How to efficiently encode data with fewer than 4 channels.
* How to efficiently encode normal maps, sRGB data, and HDR data.
* Coding equivalents to other compression formats.
* [Using rate-distortion optimization](./Docs/RDO.md).

The [ASTC Developer Guide][5] document (external link) provides a more detailed
guide for developers using the `astcenc` compressor.
Expand Down
115 changes: 115 additions & 0 deletions Source/astcenccli_compression_metrics.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// SPDX-License-Identifier: Apache-2.0
// ----------------------------------------------------------------------------
// Copyright 2026 Arm Limited
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy
// of the License at:
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// ----------------------------------------------------------------------------

/**
* @brief Measure lossless compression of ASTC block data.
*/

#if defined(ASTCENC_RDO_SIZE)

#include <algorithm>
#include <limits>
#include <string_view>

#include <zlib.h>

#include "astcenccli_internal.h"

/**
* @brief Measure a level 9 raw DEFLATE stream without retaining the compressed output.
*
* @param data The ASTC block data to compress.
* @param[out] comp_size The compressed size in bytes, excluding ZIP container metadata.
*
* @return @c true on error, @c false on success.
*/
static bool get_deflate_size(
std::string_view data,
size_t& comp_size
) {
z_stream stream {};
int status = deflateInit2(&stream, Z_BEST_COMPRESSION, Z_DEFLATED, -MAX_WBITS,
8, Z_DEFAULT_STRATEGY);
if (status != Z_OK)
{
print_error("ERROR: zlib initialization failed: %d\n", status);
return true;
}

uint8_t output[64 * 1024];
size_t offset = 0;
comp_size = 0;
do
{
// Chunk input to fit zlib's unsigned int buffer lengths
if (stream.avail_in == 0)
{
size_t count = std::min(data.size() - offset,
static_cast<size_t>(std::numeric_limits<uInt>::max()));
stream.next_in = reinterpret_cast<Bytef*>(const_cast<char*>(data.data() + offset));
stream.avail_in = static_cast<uInt>(count);
offset += count;
}

int flush = Z_NO_FLUSH;
if (offset == data.size())
{
flush = Z_FINISH;
}

stream.next_out = output;
stream.avail_out = sizeof(output);
status = deflate(&stream, flush);
comp_size += sizeof(output) - stream.avail_out;
} while (status == Z_OK);

int end_status = deflateEnd(&stream);
if ((status != Z_STREAM_END) || (end_status != Z_OK))
{
print_error("ERROR: zlib compression failed: %d\n", status);
return true;
}

return false;
}

/* See header for documentation. */
bool print_compression_metrics(
const astc_compressed_image& img
) {
std::string_view data(reinterpret_cast<const char*>(img.data.data()), img.data.size());
size_t comp_size;
bool is_compression_error = get_deflate_size(data, comp_size);
if (is_compression_error)
{
return true;
}

printf("\nStorage metrics\n");
printf("===============\n\n");

double data_sized = static_cast<double>(data.size()) / 1024.0;
double comp_sized = static_cast<double>(comp_size) / 1024.0;
double comp_percent = 100.0 * comp_sized / data_sized;

printf(" %-27s%8.1f KB\n", "ASTC data size:", data_sized);
printf(" %-27s%8.1f KB (%.2f%%)\n", "ASTC DEFLATE size:", comp_sized, comp_percent);
printf(" DEFLATE level: %8d\n", Z_BEST_COMPRESSION);
return false;
}

#endif
12 changes: 12 additions & 0 deletions Source/astcenccli_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,18 @@ bool store_cimage(
const astc_compressed_image& img,
const char* filename);

#if defined(ASTCENC_RDO_SIZE)
/**
* @brief Report the ASTC payload size before and after level 9 raw DEFLATE compression.
*
* @param img The compressed image to measure.
*
* @return @c true on error, @c false on success.
*/
bool print_compression_metrics(
const astc_compressed_image& img);
#endif

/**
* @brief Load a compressed .ktx image.
*
Expand Down
11 changes: 11 additions & 0 deletions Source/astcenccli_toplevel.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3127,6 +3127,17 @@ int astcenc_main(
}
}

#if defined(ASTCENC_RDO_SIZE)
if ((operation & ASTCENC_STAGE_COMPARE) && (config.lz_rdo_lambda > 0.0f))
{
bool is_metrics_error = print_compression_metrics(image_comp);
if (is_metrics_error)
{
return 1;
}
}
#endif

if (is_decompression_warning)
{
return 2;
Expand Down
11 changes: 11 additions & 0 deletions Source/cmake_core.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ if(${ASTCENC_CLI})
astcenccli_entry2.cpp)

add_executable(${ASTCENC_TARGET}
astcenccli_compression_metrics.cpp
astcenccli_error_metrics.cpp
astcenccli_image.cpp
astcenccli_image_external.cpp
Expand All @@ -114,6 +115,16 @@ if(${ASTCENC_CLI})
${ASTCENC_TARGET}-veneer1
${ASTCENC_TARGET}-veneer2
${ASTCENC_TARGET}-static)

if(ASTCENC_RDO_SIZE)
target_compile_definitions(${ASTCENC_TARGET}
PRIVATE
ASTCENC_RDO_SIZE)

target_link_libraries(${ASTCENC_TARGET}
PRIVATE
ZLIB::ZLIB)
endif()
endif()

macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE)
Expand Down
Loading