Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
109 changes: 5 additions & 104 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,73 +20,7 @@ env:
-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache

jobs:
changes:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
relinker: ${{ steps.filter.outputs.relinker }}
code: ${{ steps.filter.outputs.code }}
steps:
- uses: dorny/paths-filter@v4
id: filter
with:
predicate-quantifier: some-with-excludes
filters: |
relinker:
- 'core/relinker/**'
- 'core/libs/prx/libc/src/specifics/windows/**'
- 'CMakeLists.txt'
- '.github/workflows/build.yml'
code:
- '**'
- '!**/*.md'
- '!docs/**'

relinker:
needs: changes
if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.relinker == 'true') }}
runs-on: ${{ matrix.os }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
compiler: g++
- os: windows-latest
compiler: g++
- os: macos-latest
compiler: clang++
env:
WINLIBS_RELEASE: 15.2.0posix-14.0.0-ucrt-r7
WINLIBS_ZIP: winlibs-x86_64-posix-seh-gcc-15.2.0-mingw-w64ucrt-14.0.0-r7.zip
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v7
- if: runner.os == 'Windows'
id: toolchain
uses: actions/cache@v6
with:
path: C:\winlibs
key: ${{ env.WINLIBS_ZIP }}
- if: runner.os == 'Windows' && steps.toolchain.outputs.cache-hit != 'true'
run: |
curl -sSfL -o "$RUNNER_TEMP/winlibs.zip" "https://github.com/brechtsanders/winlibs_mingw/releases/download/$WINLIBS_RELEASE/$WINLIBS_ZIP"
7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs > /dev/null
- if: runner.os == 'Windows'
run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH"
- run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON
- run: cmake --build build-relinker --parallel 3
- run: ctest --test-dir build-relinker --output-on-failure --timeout 120

linux:
needs: changes
if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
Expand All @@ -96,36 +30,19 @@ jobs:
- uses: hendrikmuhs/ccache-action@v1.2.24
with:
key: linux
save: false
save: ${{ github.event_name == 'push' }}
evict-old-files: job
- name: Install lavapipe for the GPU execution tests
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers
- run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON
- name: Check that every test executable is registered with ctest
run: python3 tools/check_test_registration.py build
- run: cmake -S . -B build $CONFIGURE
- run: cmake --build build
- run: cmake --build build --target libs
- name: Save the compiler cache before the tests
if: github.event_name == 'push'
uses: actions/cache/save@v6
with:
path: ${{ github.workspace }}/.ccache
key: ccache-linux-${{ github.run_id }}-${{ github.run_attempt }}
- run: ctest --test-dir build --output-on-failure --timeout 120 ${{ github.event_name == 'push' && '--repeat until-fail:3' || '' }} --parallel 4 --output-log ctest.log
- run: ctest --test-dir build --output-on-failure --timeout 120 --repeat until-fail:3
env:
ANYPS5_REQUIRE_VULKAN: 1
GALLIUM_OVERRIDE_CPU_CAPS: avx
- name: Report tests slower than 30 s
if: ${{ !cancelled() }}
shell: bash
run: |
[ -f ctest.log ] || exit 0
awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t }
END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log

windows:
needs: changes
if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') }}
runs-on: windows-latest
timeout-minutes: 45
env:
Expand All @@ -152,29 +69,13 @@ jobs:
- uses: hendrikmuhs/ccache-action@v1.2.24
with:
key: windows
save: false
save: ${{ github.event_name == 'push' }}
evict-old-files: job
- shell: cmd
run: cmake -S . -B build %CONFIGURE%
- name: Check that every test executable is registered with ctest
shell: cmd
run: python tools/check_test_registration.py build
- shell: cmd
run: cmake --build build
- shell: cmd
run: cmake --build build --target libs
- name: Save the compiler cache before the tests
if: github.event_name == 'push'
uses: actions/cache/save@v6
with:
path: ${{ github.workspace }}\.ccache
key: ccache-windows-${{ github.run_id }}-${{ github.run_attempt }}
- shell: cmd
run: ctest --test-dir build --output-on-failure --timeout 120 ${{ github.event_name == 'push' && '--repeat until-fail:3' || '' }} --parallel 4 --output-log ctest.log
- name: Report tests slower than 30 s
if: ${{ !cancelled() }}
shell: bash
run: |
[ -f ctest.log ] || exit 0
awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t }
END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log
run: ctest --test-dir build --output-on-failure --timeout 120 --repeat until-fail:3
22 changes: 0 additions & 22 deletions .github/workflows/conventions.yml

This file was deleted.

1 change: 1 addition & 0 deletions .github/workflows/pr-overlap.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ jobs:
python3 tools/pr_overlap.py --base "${{ github.event.repository.default_branch }}" --out "$out"
marker='<!-- pr-overlap -->'
for file in "$out"/*.md; do
[ -e "$file" ] || continue
pr=$(basename "$file" .md)
comment=$(gh api "repos/$GH_REPO/issues/$pr/comments" --paginate \
--jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | [.id, .body] | @json" | head -n1)
Expand Down
4 changes: 0 additions & 4 deletions .github/workflows/progress-report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@ name: Progress report

on:
pull_request:
paths:
- 'core/libs/prx/**'
- 'tools/progress.py'
- '.github/workflows/progress-report.yml'

permissions:
contents: read
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/progress.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ jobs:
steps:
- uses: actions/checkout@v7
- run: python3 tools/progress.py out
- uses: actions/configure-pages@v5
- uses: actions/upload-pages-artifact@v5
with:
path: out
Expand Down
3 changes: 0 additions & 3 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,3 @@
[submodule "3rdparty/stb"]
path = 3rdparty/stb
url = https://github.com/nothings/stb.git
[submodule "3rdparty/libjpeg-turbo"]
path = 3rdparty/libjpeg-turbo
url = https://github.com/libjpeg-turbo/libjpeg-turbo.git
1 change: 0 additions & 1 deletion 3rdparty/libjpeg-turbo
Submodule libjpeg-turbo deleted from c85e6b
22 changes: 2 additions & 20 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ function(add_test_executable target)
endfunction()

option(BUILD_TESTING "Build and register relinker tests" OFF)
option(ANYPS5_RELINKER_ONLY "Build the relinker without system libraries or third-party dependencies" OFF)
if(BUILD_TESTING)
enable_testing()
if(MINGW)
Expand All @@ -24,30 +23,13 @@ if(BUILD_TESTING)
endif()

option(ANYPS5_ENABLE_SPIRV_TOOLS "Enable SPIRV-Tools validation and optimization" OFF)
option(APS5_ENABLE_TIMING_LOG "Write frame timings asynchronously to frame-timing.log" OFF)
option(APS5_ENABLE_TIMING_LOG "Enable frame timing logging" OFF)
add_compile_definitions(APS5_ENABLE_TIMING_LOG=$<BOOL:${APS5_ENABLE_TIMING_LOG}>)

set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

if(NOT APPLE)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")
endif()

if(BUILD_TESTING)
find_package(Python3 QUIET COMPONENTS Interpreter)
if(Python3_Interpreter_FOUND)
add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
endif()
endif()

if(ANYPS5_RELINKER_ONLY)
add_subdirectory(core/relinker)
return()
endif()
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++")

set(SDL_SHARED OFF CACHE BOOL "" FORCE)
set(SDL_STATIC ON CACHE BOOL "" FORCE)
Expand Down
13 changes: 1 addition & 12 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,17 @@

- Follow the [coding conventions](docs/dev/CONVENTIONS.md): naming, no comments except [technical debt](docs/dev/TechnicalDebt.md), Conventional Commits.
- Every function either does exactly what it is supposed to or throws. Unimplemented exports call `NotImplemented_nid_no_patch(__func__)` (see [libSceAudioIn](core/libs/prx/libSceAudioIn/Export.cpp)).
- Pull requests that add or change shader instruction semantics say where they come from: measured on hardware (which GPU and what was checked, e.g. with the [hardware oracle](docs/dev/HW_ORACLE.md)) or the exact source (ISA section, LLVM, ACO or Mesa file). Reviewers check the semantics on hardware. Cases the source doesn't settle throw, and behaviour not verified on hardware is recorded in [technical debt](docs/dev/TechnicalDebt.md).
- A silent stub is allowed only when it unblocks a title and only affects the UI; add it to [silent stubs](docs/dev/TechnicalDebt.md#silent-stubs).
- Missing imports fail at startup with a clear error; don't replace them with fallbacks that keep running.
- Implement the general behaviour of a function, not what one title happens to need.
- Don't add replacements for modules that titles ship themselves in `sce_module/`, `sce_modules/` or `prx/`: engine or middleware modules (Cohtml, FMOD, GOG Galaxy) and SDK libraries that only wrap other system libraries (NpCppWebApi over NpWebApi2). The relinker converts and loads the title's own module, and a host library with the same name is left out of `DT_NEEDED`. Only system libraries, which reach the kernel or the hardware, are reimplemented in [core/libs/prx](core/libs/prx).
- Avoid non-standard extensions (`__attribute__`, etc.) where standard C++ is enough. Helper symbols that must not become NIDs use the `_nid_no_patch` or `_nid_no_patch_cut` suffix.
- The relinker uses only the C++20 standard library.
- Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system.
- Don't add tests that only check that a symbol is exported: a missing export already fails at startup.
- Each test finishes in 30 s on the CI runners, Vulkan tests included (Linux runs them on lavapipe), counting the first run with no shader cache. ctest runs in parallel there, so a slow test competes for the cores and times out on some runs and not others. Split or shrink a test that gets close instead of raising its timeout. CI reports every test over 30 s as a warning on the pull request.

## Build and test

Toolchains are listed in the [build instructions](docs/dev/BUILD.md).

For relinker changes, use the [relinker-only build](docs/dev/BUILD.md#relinker-only) to build and test without third-party submodules, system libraries or a GPU. It also works on macOS. Changes to system libraries or shaders still need the full build and relevant runtime tests.
Toolchains are listed in the [README](README.md#build).

```
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON
Expand All @@ -29,12 +24,6 @@ ctest --test-dir build --output-on-failure

Python 3 is optional; without it some relinker tests are not registered.

The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing:

```
python3 tools/check_conventions.py --base origin/main
```

## Branches and pull requests

The [pull request template](.github/pull_request_template.md) is the checklist for these rules.
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Tool for automatic executables porting to Linux and Windows.

Includes a [relinker](core/relinker) that converts executable to the target system's native format and implementations of [system prx libraries](core/libs/prx) suitable for dynamic linking. No emulation or separate runtime process.

[Usage](docs/user/USAGE.md), [Build instructions](docs/dev/BUILD.md), [Architecture](docs/dev/ARCHITECTURE.md), [Technical debt of the project](docs/dev/TechnicalDebt.md), [code style conventions](docs/dev/CONVENTIONS.md), [contributing](CONTRIBUTING.md)
[Usage](docs/user/USAGE.md), [Build instructions](docs/dev/BUILD.md), [Technical debt of the project](docs/dev/TechnicalDebt.md), [code style conventions](docs/dev/CONVENTIONS.md), [contributing](CONTRIBUTING.md)

## Status

Expand Down
48 changes: 1 addition & 47 deletions core/Decoder/Jpeg/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,10 @@ function(add_jpeg_decoder target runtimeTarget)
message(FATAL_ERROR "JPEG decoder requires an existing runtime target: ${runtimeTarget}")
endif()

include(ExternalProject)
if(NOT TARGET jpeg_turbo_external)
set(jpeg_turbo_prefix "${CMAKE_BINARY_DIR}/libjpeg-turbo")
if(WIN32)
if(MSVC)
set(jpeg_turbo_library "${jpeg_turbo_prefix}/lib/turbojpeg-static.lib")
else()
set(jpeg_turbo_library "${jpeg_turbo_prefix}/lib/libturbojpeg.a")
endif()
else()
set(jpeg_turbo_library "${jpeg_turbo_prefix}/lib/libturbojpeg.a")
endif()
file(MAKE_DIRECTORY "${jpeg_turbo_prefix}/include")
ExternalProject_Add(jpeg_turbo_external
SOURCE_DIR "${PROJECT_SOURCE_DIR}/3rdparty/libjpeg-turbo"
BINARY_DIR "${CMAKE_BINARY_DIR}/libjpeg-turbo-build"
INSTALL_DIR "${jpeg_turbo_prefix}"
CMAKE_ARGS
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${jpeg_turbo_prefix}
-DBUILD_SHARED_LIBS=OFF
-DENABLE_STATIC=ON
-DENABLE_SHARED=OFF
-DWITH_SIMD=OFF
-DWITH_TURBOJPEG=ON
-DWITH_TOOLS=OFF
-DWITH_TESTS=OFF
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_C_VISIBILITY_PRESET=hidden
-DCMAKE_VISIBILITY_INLINES_HIDDEN=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
-DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
BUILD_BYPRODUCTS
"${jpeg_turbo_library}"
)
add_library(jpeg_turbo STATIC IMPORTED GLOBAL)
set_target_properties(jpeg_turbo PROPERTIES
IMPORTED_LOCATION "${jpeg_turbo_library}"
INTERFACE_INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/3rdparty/libjpeg-turbo/src;${jpeg_turbo_prefix}/include"
)
add_dependencies(jpeg_turbo jpeg_turbo_external)
endif()

add_library(${target} STATIC ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/src/Jpeg.cpp)
target_include_directories(${target} PUBLIC ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/include)
target_include_directories(${target} SYSTEM PRIVATE ${PROJECT_SOURCE_DIR}/3rdparty/stb)
target_link_libraries(${target} PRIVATE ${runtimeTarget} jpeg_turbo)
target_link_libraries(${target} PRIVATE ${runtimeTarget})
set_target_properties(${target} PROPERTIES
CXX_EXTENSIONS OFF
CXX_VISIBILITY_PRESET hidden
Expand Down
9 changes: 1 addition & 8 deletions core/Decoder/Jpeg/include/Decoder/Jpeg.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,8 @@ struct Header {
std::uint32_t channels;
};

enum class Sampling {
Yuv444,
Yuv422,
Yuv420,
};

std::vector<std::uint8_t> Encode(std::span<const std::uint8_t> pixels, std::uint32_t width, std::uint32_t height,
std::uint32_t channels, int quality, Sampling sampling = Sampling::Yuv444,
std::uint32_t restartBlocks = 0, std::uint32_t restartRows = 0);
std::uint32_t channels, int quality);

std::optional<Header> ParseHeader(std::span<const std::uint8_t> jpeg);

Expand Down
Loading
Loading