feat(agc): implement image tables loaded through 64-bit pointers

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -66,6 +66,12 @@ class Driver {
     static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
     static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);
     DrawEntryCounters DrawCacheCounters();
+    struct DispatchCacheFootprint {
+        std::size_t entries = 0;
+        std::uint64_t variantBytes = 0;
+    };
+    DispatchCacheFootprint DispatchCacheSize();
+    static std::uint64_t DispatchCacheBytes();
 
 private:
     friend class SampledReadScope;

core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp

@@ -99,6 +99,8 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
 void BumpCollectEpoch();
 std::uint64_t CollectEpoch();
 std::uint64_t CollectEpochBumps();
+std::uint64_t ThreadCollectEpoch();
+std::uint64_t UnwatchSerial();
 std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes);
 // The tracker's current generation (every collect and MarkWritten bumps it): a stamp taken after
 // a set of driver stores, for UnchangedSinceCollected to compare against later. Read under the

core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp

@@ -85,8 +85,11 @@ class ShaderMemory {
     };
 
     static bool read(void* context, std::uint64_t address, std::uint32_t* value);
+    static bool readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count);
     static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes);
+    static bool pending(void* context, std::uint64_t address, std::uint64_t bytes);
     Page& page(std::uint64_t base);
+    [[nodiscard]] bool overlapsInitial(std::uint64_t begin, std::uint64_t end) const;
 
     // Regions given at construction (the registered shader's code and header), referenced as given:
     // the caller keeps them alive for as long as the capture is used.

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCache.cpp

@@ -45,6 +45,20 @@ std::size_t Driver::dispatchCacheEntries() {
     return entries;
 }
 
+std::uint64_t Driver::DispatchCacheBytes() {
+    static const std::uint64_t bytes = [] {
+        const char* text = std::getenv("APS5_DISPATCH_CACHE_BYTES");
+        const auto parsed = text != nullptr ? std::strtoull(text, nullptr, 10) : 0ull;
+        return parsed != 0 ? static_cast<std::uint64_t>(parsed) : std::uint64_t{256} << 20u;
+    }();
+    return bytes;
+}
+
+Driver::DispatchCacheFootprint Driver::DispatchCacheSize() {
+    std::lock_guard lock(dispatchCacheMutex);
+    return {dispatchCache.size(), dispatchCacheVariantBytes};
+}
+
 std::uint64_t Driver::variantBytes(const DispatchVariant& variant) {
     return sizeof(DispatchVariant) + variant.words.size() * sizeof(std::uint32_t) + variant.runs.size() * sizeof(std::pair<std::uint64_t, std::uint64_t>) + variant.captured.size() * sizeof(ShaderRecompiler::MemoryRegion) + (variant.vertexInfo != nullptr ? sizeof(ShaderRecompiler::ShaderVertexStageInfo) : 0);
 }

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp

@@ -1,13 +1,17 @@
 #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
 #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
 #include "Optimization/ResourceProgram.hpp"
+#include "ImageTableAbi.hpp"
+#include <algorithm>
 #include <cstdlib>
 #include <cstring>
 
 namespace AgcDriver::DriverDetail {
 
 bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled) {
-    return compiled.poisonedSrtReads == 0u;
+    return compiled.poisonedSrtReads == 0u && std::none_of(compiled.imageTablePoison.begin(), compiled.imageTablePoison.end(), [](const ShaderRecompiler::ImageTableEntryPoison& poison) {
+        return poison.reason == static_cast<std::uint32_t>(ShaderRecompiler::ImageTableAbi::PoisonReason::Unmapped) || poison.reason == static_cast<std::uint32_t>(ShaderRecompiler::ImageTableAbi::PoisonReason::Inactive);
+    });
 }
 
 void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr<const ShaderSnapshot>& registeredShader, std::uint64_t forgetAtCapture, std::span<const ShaderRecompiler::MemoryRegion> memory, const std::shared_ptr<ShaderMemory>& shaderMemory, const std::vector<ShaderRecompiler::MemoryRegion>& captured, const std::shared_ptr<const ShaderRecompiler::ResourceCapture>& capture, const std::shared_ptr<const ShaderRecompiler::RecompileResult>& compiledResult, const std::shared_ptr<DispatchEntry>& missedEntry, bool missedDiffering, std::shared_ptr<DispatchVariant>& attachVariant, DispatchPhaseTiming& phaseTiming) {
@@ -113,10 +117,13 @@ void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDis
             }
 
             static const bool lru = std::getenv("APS5_NO_DISPATCH_LRU") == nullptr;
-            if (dispatchCache.size() > dispatchCacheEntries()) {
+            const auto over = [&] { return dispatchCache.size() > dispatchCacheEntries() || (dispatchCache.size() > 1 && dispatchCacheVariantBytes > DispatchCacheBytes()); };
+            if (over()) {
                 if (lru) {
-                    eraseDispatchEntry(dispatchCache.find(dispatchOrder.back()));
-                    ++dispatchCacheEvictions;
+                    while (over()) {
+                        eraseDispatchEntry(dispatchCache.find(dispatchOrder.back()));
+                        ++dispatchCacheEvictions;
+                    }
                 } else {
                     dispatchCacheEvictions += dispatchCache.size();
                     dispatchCache.clear();

core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp

@@ -1213,6 +1213,14 @@ void BumpCollectEpoch() {
     collectEpochBumps.fetch_add(1, std::memory_order_relaxed);
 }
 
+std::uint64_t ThreadCollectEpoch() {
+    return collectMemoEnabled() ? threadCollectEpoch : 0;
+}
+
+std::uint64_t UnwatchSerial() {
+    return unwatchSerial.load(std::memory_order_acquire);
+}
+
 std::uint64_t CollectEpochBumps() {
     return collectEpochBumps.load(std::memory_order_relaxed);
 }

core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp

@@ -59,6 +59,7 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
     checkFeature(spv::CapabilitySampledImageArrayNonUniformIndexing, sampledNonUniform, "shaderSampledImageArrayNonUniformIndexing");
     checkFeature(spv::CapabilityStorageImageArrayNonUniformIndexing, storageNonUniform, "shaderStorageImageArrayNonUniformIndexing");
     checkFeature(spv::CapabilityShaderNonUniform, sampledNonUniform || storageNonUniform, "descriptor indexing");
+    checkFeature(spv::CapabilityRuntimeDescriptorArray, indexing != nullptr && indexing->runtimeDescriptorArray == VK_TRUE, "runtimeDescriptorArray");
     const bool barycentricEnabled = barycentric != nullptr && barycentric->fragmentShaderBarycentric == VK_TRUE;
     const bool meshEnabled = mesh != nullptr && mesh->meshShader == VK_TRUE;
     checkFeature(spv::CapabilityFragmentBarycentricKHR, barycentricEnabled, "fragmentShaderBarycentric");

core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp

@@ -201,6 +201,43 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
     return true;
 }
 
+bool ShaderMemory::overlapsInitial(std::uint64_t begin, std::uint64_t end) const {
+    if (initial.empty()) return false;
+    const auto after = initial.lower_bound(begin);
+    if (after != initial.end() && after->first < end) return true;
+    return after != initial.begin() && std::prev(after)->first + std::prev(after)->second.size() > begin;
+}
+
+bool ShaderMemory::readWords(void* context, std::uint64_t address, std::uint32_t* values, std::uint64_t count) {
+    auto& self = *static_cast<ShaderMemory*>(context);
+    while (count != 0) {
+        const auto base = address & ~static_cast<std::uint64_t>(PageBytes - 1);
+        const auto index = static_cast<std::size_t>((address % PageBytes) / sizeof(std::uint32_t));
+        const auto words = static_cast<std::size_t>(std::min<std::uint64_t>(count, PageWords - index));
+        const auto bytes = static_cast<std::uint64_t>(words) * sizeof(std::uint32_t);
+        Page* page = nullptr;
+        if (address % sizeof(std::uint32_t) == 0 && address <= std::numeric_limits<std::uint64_t>::max() - bytes && !self.overlapsInitial(address, address + bytes)) page = &self.page(base);
+        if (page != nullptr && page->valid.all()) {
+            std::memcpy(values, page->words.data() + index, bytes);
+            std::bitset<PageWords> mask;
+            mask.set();
+            mask >>= PageWords - words;
+            mask <<= index;
+            page->read |= mask;
+            page->recent |= mask;
+            CaptureTotals().reads += words;
+        } else {
+            for (std::size_t word = 0; word < words; ++word) {
+                if (!read(context, address + word * sizeof(std::uint32_t), values + word)) return false;
+            }
+        }
+        address += bytes;
+        values += words;
+        count -= words;
+    }
+    return true;
+}
+
 bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_t bytes) {
     auto& self = *static_cast<ShaderMemory*>(context);
     if (bytes == 0 || address > std::numeric_limits<std::uint64_t>::max() - bytes) return false;
@@ -223,6 +260,13 @@ bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_
     return true;
 }
 
+bool ShaderMemory::pending(void* context, std::uint64_t address, std::uint64_t bytes) {
+    const auto& self = *static_cast<const ShaderMemory*>(context);
+    if (self.pendingWrite == nullptr || bytes == 0) return false;
+    const auto policy = self.pendingWrite(address, static_cast<std::size_t>(bytes), {});
+    return policy == PendingWrite::Sync || policy == PendingWrite::VerifyRaw || policy == PendingWrite::VerifyKnownValue;
+}
+
 ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() {
     auto& totals = CaptureTotals();
     return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)};
@@ -256,6 +300,8 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
     runtime.readMemory = &read;
     runtime.readSpecializationMemory = &read;
     runtime.accessible = &accessible;
+    runtime.pendingWrite = &pending;
+    runtime.readWords = &readWords;
     auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime);
     if (profile) {
         totals.captureNanoseconds += NanosecondsSince(started);

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -1477,9 +1477,9 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
         state->descriptorIndexing = descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing == VK_TRUE;
         state->storageBufferUpdateAfterBind = descriptorIndexingFeatures.descriptorBindingStorageBufferUpdateAfterBind == VK_TRUE;
-        state->descriptorBindingPartiallyBound = compileOnly && descriptorIndexingFeatures.descriptorBindingPartiallyBound == VK_TRUE;
-        state->descriptorBindingVariableDescriptorCount = compileOnly && descriptorIndexingFeatures.descriptorBindingVariableDescriptorCount == VK_TRUE;
-        state->runtimeDescriptorArray = compileOnly && descriptorIndexingFeatures.runtimeDescriptorArray == VK_TRUE;
+        state->descriptorBindingPartiallyBound = descriptorIndexingFeatures.descriptorBindingPartiallyBound == VK_TRUE;
+        state->descriptorBindingVariableDescriptorCount = descriptorIndexingFeatures.descriptorBindingVariableDescriptorCount == VK_TRUE;
+        state->runtimeDescriptorArray = descriptorIndexingFeatures.runtimeDescriptorArray == VK_TRUE;
         if (state->storageBufferUpdateAfterBind) {
             VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &state->descriptorIndexingProperties};
             state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
@@ -3258,6 +3258,11 @@ Graphics::Context VulkanDevice::buildContext() const {
     context.descriptorIndexing = state->descriptorIndexing;
     context.descriptorIndexingLimits = state->descriptorIndexingProperties;
     context.imageInt64Atomics = state->imageInt64Atomics;
+    context.runtimeDescriptorArray = state->runtimeDescriptorArray;
+    context.partiallyBound = state->descriptorBindingPartiallyBound;
+    context.variableDescriptorCount = state->descriptorBindingVariableDescriptorCount;
+    context.imageTableCapacity = ImageTableCapacity(context);
+    context.samplerTableCapacity = SamplerTableCapacity(context);
     context.geometryShader = state->geometryShader;
     context.sampleRateShading = state->sampleRateShading;
     context.shaderClipDistance = state->shaderClipDistance;

core/libs/prx/libSceAgcDriver/Graphics/include/BdaResources.hpp

@@ -2,6 +2,10 @@
 #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_BDARESOURCES_HPP
 
 #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
+#include "Recompiler.hpp"
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace AgcDriver::Graphics {
 
@@ -14,6 +18,13 @@ class BdaResources {
     VkDescriptorBufferInfo Table() const;
     VkDescriptorBufferInfo Fault() const;
     void CheckFault() const;
+    struct ImageTableFaults {
+        std::uint32_t stage = 0;
+        std::uint64_t shader = 0;
+        std::vector<ShaderRecompiler::ImageTableEntryPoison> poison;
+        std::vector<std::string> messages;
+    };
+    void SetImageTables(std::vector<ImageTableFaults> tables, std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges);
     // APS5_PROFILE_DRAW: page tables served from the per-device cache and built anew (cumulative),
     // and how many recent tables the cache holds right now.
     struct TableCacheStats {
@@ -44,6 +55,8 @@ class BdaResources {
     std::shared_ptr<Buffer> table;
     std::unique_ptr<Buffer> fault;
     std::size_t tableBytes = 0;
+    std::vector<ImageTableFaults> imageTables;
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> imageTableRanges;
 };
 
 }

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -73,6 +73,10 @@ inline void Require(bool condition, const char* reason) {
     if (!condition) throw std::runtime_error(std::string("AGC graphics: ") + reason);
 }
 
+struct DescriptorRejected : std::runtime_error {
+    using std::runtime_error::runtime_error;
+};
+
 // Device entry points resolved once per device (VulkanDevice's State fills it after setup): the
 // loader's vkGetDeviceProcAddr is a name lookup under its global mutex per call, paid at every
 // record site otherwise. Null members (tests, APS5_NO_PROC_TABLE=1) resolve per call (Resolved).
@@ -196,6 +200,11 @@ struct Context {
     bool descriptorIndexing = false;
     VkPhysicalDeviceDescriptorIndexingPropertiesEXT descriptorIndexingLimits{};
     bool imageInt64Atomics = false;
+    bool runtimeDescriptorArray = false;
+    bool partiallyBound = false;
+    bool variableDescriptorCount = false;
+    std::uint32_t imageTableCapacity = 0;
+    std::uint32_t samplerTableCapacity = 0;
     bool geometryShader = false;
     bool sampleRateShading = false;
     bool shaderClipDistance = false;
@@ -298,6 +307,9 @@ inline void EndRenderPass(const Context& context, VkCommandBuffer commands) {
     else context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands);
 }
 
+std::uint32_t ImageTableCapacity(const Context& context);
+std::uint32_t SamplerTableCapacity(const Context& context);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp

@@ -16,6 +16,7 @@ class StorageTexture;
 
 VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target);
 bool SampleLocationsCompatibleDepth(const Context& context, VkFormat format, std::uint32_t samples);
+std::uint64_t DepthSurfacesSerial();
 std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel);
 void ClearDepthSurfaces(VkDevice device);
 bool DepthSurfaceAt(std::uint64_t address);

core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp

@@ -124,7 +124,7 @@ bool ValidateSampleLocations(const Context& context, const State& state);
 void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment);
 bool ShadersWriteMemory(std::span<const CompiledShader> shaders);
 // Returns the color attachment locations the pixel shader writes.
-std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false);
+std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false, bool shaderClipDistance = false, bool shaderCullDistance = false, bool viewportIndexLayer = false, bool runtimeDescriptorArray = false);
 
 }
 

core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp

@@ -24,6 +24,8 @@ class Sampler {
     bool RequiresFilterMinmax() const;
     bool ReadsOpaqueBlackBorder() const;
     bool RequiresNonSeamlessCube() const;
+    static void Validate(const Context& context, const GuestSamplerResource& descriptor);
+    static bool ReadsOpaqueBlackBorder(const GuestSamplerResource& descriptor);
 
 private:
     void release() noexcept;