diff --git a/include/RAJA/policy/cuda/policy.hpp b/include/RAJA/policy/cuda/policy.hpp index d521dede10..114c6fe47b 100644 --- a/include/RAJA/policy/cuda/policy.hpp +++ b/include/RAJA/policy/cuda/policy.hpp @@ -652,6 +652,143 @@ struct IndexSize {} }; +// Class to help cache thread indices or not based on template arg +template +struct ThreadIndices +{ + template + RAJA_DEVICE constexpr cuda_dim_member_t get_threadIdx() const + { + return ::RAJA::internal::CudaDimHelper::get(threadIdx); + } +}; + +template<> +struct ThreadIndices +{ + dim3 m_threadIdx; + + RAJA_HOST_DEVICE ThreadIndices() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_threadIdx(threadIdx) +#endif + {} + + template + RAJA_DEVICE constexpr cuda_dim_member_t get_threadIdx() const + { + return ::RAJA::internal::CudaDimHelper::get(m_threadIdx); + } +}; + +// Class to help cache block indices or not based on template arg +template +struct BlockIndices +{ + template + RAJA_DEVICE constexpr cuda_dim_member_t get_blockIdx() const + { + return ::RAJA::internal::CudaDimHelper::get(blockIdx); + } +}; + +template<> +struct BlockIndices +{ + dim3 m_blockIdx; + + RAJA_HOST_DEVICE BlockIndices() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_blockIdx(blockIdx) +#endif + {} + + template + RAJA_DEVICE constexpr cuda_dim_member_t get_blockIdx() const + { + return ::RAJA::internal::CudaDimHelper::get(m_blockIdx); + } +}; + +// Class to help cache block dimensions or not based on template arg +template +struct BlockDimensions +{ + template + RAJA_DEVICE constexpr cuda_dim_member_t get_blockDim() const + { + return ::RAJA::internal::CudaDimHelper::get(blockDim); + } +}; + +template<> +struct BlockDimensions +{ + dim3 m_blockDim; + + RAJA_HOST_DEVICE BlockDimensions() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_blockDim(blockDim) +#endif + {} + + template + RAJA_DEVICE constexpr cuda_dim_member_t get_blockDim() const + { + return ::RAJA::internal::CudaDimHelper::get(m_blockDim); + } +}; + +// Class to help cache grid dimensions or not based on template arg +template +struct GridDimensions +{ + template + RAJA_DEVICE constexpr cuda_dim_member_t get_gridDim() const + { + return ::RAJA::internal::CudaDimHelper::get(gridDim); + } +}; + +template<> +struct GridDimensions +{ + dim3 m_gridDim = gridDim; + + RAJA_HOST_DEVICE GridDimensions() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_gridDim(gridDim) +#endif + {} + + template + RAJA_DEVICE constexpr cuda_dim_member_t get_gridDim() const + { + return ::RAJA::internal::CudaDimHelper::get(m_gridDim); + } +}; + +// Class to help cache indices and dimensions or not based on template args +template +struct IndicesAndDims : ThreadIndices, + BlockIndices, + BlockDimensions, + GridDimensions +{}; + +// Nothing cached +using NonCachedIndicesAndDims = IndicesAndDims; + +// threadIdx and blockDim cached, rest not cached +using CachedThreadIndicesAndBlockDims = + IndicesAndDims; + +// threadIdx, blockDim, blockIdx, gridDim cached +using AllCachedIndicesAndDims = IndicesAndDims; + /// Type representing thread indexing within a grid /// It has various specializations that optimize specific patterns @@ -666,18 +803,18 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)) + + return static_cast(idxNDims.template get_threadIdx()) + static_cast(block_size) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size) * static_cast(grid_size); } @@ -692,15 +829,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(grid_size); } @@ -715,15 +853,16 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size); } @@ -736,14 +875,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -758,22 +899,20 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)) + - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_threadIdx()) + + static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)) * + return static_cast(idxNDims.template get_blockDim()) * static_cast(grid_size); } }; @@ -785,18 +924,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)); + return static_cast(idxNDims.template get_blockDim()); } }; @@ -809,22 +948,21 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)) + + return static_cast(idxNDims.template get_threadIdx()) + static_cast(block_size) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(gridDim)); + static_cast(idxNDims.template get_gridDim()); } }; @@ -835,18 +973,18 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_gridDim()); } }; @@ -857,24 +995,21 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)) + - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_threadIdx()) + + static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)) * - static_cast( - ::RAJA::internal::CudaDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_gridDim()); } }; @@ -888,15 +1023,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(grid_size); } @@ -909,14 +1045,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -929,18 +1067,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_gridDim()); } }; @@ -954,15 +1092,16 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size); } @@ -975,14 +1114,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -995,18 +1136,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::CudaDimHelper::get(blockDim)); + return static_cast(idxNDims.template get_blockDim()); } }; @@ -1018,14 +1159,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -1036,17 +1179,19 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index(); + return x_index::template index(idxNDims); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size(); + return x_index::template size(idxNDims); } }; @@ -1055,18 +1200,22 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index() + - x_index::template size() * (y_index::template index()); + return x_index::template index(idxNDims) + + x_index::template size(idxNDims) * + (y_index::template index(idxNDims)); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size() * y_index::template size(); + return x_index::template size(idxNDims) * + y_index::template size(idxNDims); } }; @@ -1075,38 +1224,43 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index() + - x_index::template size() * - (y_index::template index() + - y_index::template size() * - z_index::template index()); + return x_index::template index(idxNDims) + + x_index::template size(idxNDims) * + (y_index::template index(idxNDims) + + y_index::template size(idxNDims) * + z_index::template index(idxNDims)); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size() * y_index::template size() * - z_index::template size(); + return x_index::template size(idxNDims) * + y_index::template size(idxNDims) * + z_index::template size(idxNDims); } }; template struct IndexDivide { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return indexer::template index() / static_cast(divisor); + return indexer::template index(idxNDims) / static_cast(divisor); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return RAJA_DIVIDE_CEILING_INT(indexer::template size(), + return RAJA_DIVIDE_CEILING_INT(indexer::template size(idxNDims), static_cast(divisor)); } }; @@ -1114,14 +1268,16 @@ struct IndexDivide template struct IndexModulo { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return indexer::template index() % static_cast(divisor); + return indexer::template index(idxNDims) % static_cast(divisor); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(divisor); } diff --git a/include/RAJA/policy/hip/policy.hpp b/include/RAJA/policy/hip/policy.hpp index e7c3ca0672..0951187ba7 100644 --- a/include/RAJA/policy/hip/policy.hpp +++ b/include/RAJA/policy/hip/policy.hpp @@ -644,6 +644,143 @@ struct IndexSize {} }; +// Class to help cache thread indices or not based on template arg +template +struct ThreadIndices +{ + template + RAJA_DEVICE constexpr hip_dim_member_t get_threadIdx() const + { + return ::RAJA::internal::HipDimHelper::get(threadIdx); + } +}; + +template<> +struct ThreadIndices +{ + dim3 m_threadIdx; + + RAJA_HOST_DEVICE ThreadIndices() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_threadIdx(threadIdx) +#endif + {} + + template + RAJA_DEVICE constexpr hip_dim_member_t get_threadIdx() const + { + return ::RAJA::internal::HipDimHelper::get(m_threadIdx); + } +}; + +// Class to help cache block indices or not based on template arg +template +struct BlockIndices +{ + template + RAJA_DEVICE constexpr hip_dim_member_t get_blockIdx() const + { + return ::RAJA::internal::HipDimHelper::get(blockIdx); + } +}; + +template<> +struct BlockIndices +{ + dim3 m_blockIdx; + + RAJA_HOST_DEVICE BlockIndices() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_blockIdx(blockIdx) +#endif + {} + + template + RAJA_DEVICE constexpr hip_dim_member_t get_blockIdx() const + { + return ::RAJA::internal::HipDimHelper::get(m_blockIdx); + } +}; + +// Class to help cache block dimensions or not based on template arg +template +struct BlockDimensions +{ + template + RAJA_DEVICE constexpr hip_dim_member_t get_blockDim() const + { + return ::RAJA::internal::HipDimHelper::get(blockDim); + } +}; + +template<> +struct BlockDimensions +{ + dim3 m_blockDim; + + RAJA_HOST_DEVICE BlockDimensions() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_blockDim(blockDim) +#endif + {} + + template + RAJA_DEVICE constexpr hip_dim_member_t get_blockDim() const + { + return ::RAJA::internal::HipDimHelper::get(m_blockDim); + } +}; + +// Class to help cache grid dimensions or not based on template arg +template +struct GridDimensions +{ + template + RAJA_DEVICE constexpr hip_dim_member_t get_gridDim() const + { + return ::RAJA::internal::HipDimHelper::get(gridDim); + } +}; + +template<> +struct GridDimensions +{ + dim3 m_gridDim = gridDim; + + RAJA_HOST_DEVICE GridDimensions() +#if defined(RAJA_GPU_DEVICE_COMPILE_PASS_ACTIVE) + : m_gridDim(gridDim) +#endif + {} + + template + RAJA_DEVICE constexpr hip_dim_member_t get_gridDim() const + { + return ::RAJA::internal::HipDimHelper::get(m_gridDim); + } +}; + +// Class to help cache indices and dimensions or not based on template args +template +struct IndicesAndDims : ThreadIndices, + BlockIndices, + BlockDimensions, + GridDimensions +{}; + +// Nothing cached +using NonCachedIndicesAndDims = IndicesAndDims; + +// threadIdx and blockDim cached, rest not cached +using CachedThreadIndicesAndBlockDims = + IndicesAndDims; + +// threadIdx, blockDim, blockIdx, gridDim cached +using AllCachedIndicesAndDims = IndicesAndDims; + /// Type representing thread indexing within a grid /// It has various specializations that optimize specific patterns @@ -658,18 +795,18 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)) + + return static_cast(idxNDims.template get_threadIdx()) + static_cast(block_size) * - static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size) * static_cast(grid_size); } @@ -684,15 +821,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(grid_size); } @@ -707,15 +845,16 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size); } @@ -728,14 +867,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -750,22 +891,20 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)) + - static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)) * - static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_threadIdx()) + + static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)) * + return static_cast(idxNDims.template get_blockDim()) * static_cast(grid_size); } }; @@ -777,18 +916,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)); + return static_cast(idxNDims.template get_blockDim()); } }; @@ -801,21 +940,21 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)) + + return static_cast(idxNDims.template get_threadIdx()) + static_cast(block_size) * - static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size) * - static_cast(::RAJA::internal::HipDimHelper::get(gridDim)); + static_cast(idxNDims.template get_gridDim()); } }; @@ -826,17 +965,18 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast(::RAJA::internal::HipDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_gridDim()); } }; @@ -847,23 +987,21 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)) + - static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)) * - static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_threadIdx()) + + static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)) * - static_cast(::RAJA::internal::HipDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_blockDim()) * + static_cast(idxNDims.template get_gridDim()); } }; @@ -877,15 +1015,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = GRID_SIZE; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(grid_size); } @@ -898,14 +1037,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = 1; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -918,17 +1059,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = named_usage::unspecified; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockIdx)); + return static_cast(idxNDims.template get_blockIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast(::RAJA::internal::HipDimHelper::get(gridDim)); + return static_cast(idxNDims.template get_gridDim()); } }; @@ -942,15 +1084,16 @@ struct IndexGlobal static constexpr int block_size = BLOCK_SIZE; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static constexpr IdxT size() + template + RAJA_DEVICE static constexpr IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(block_size); } @@ -963,14 +1106,16 @@ struct IndexGlobal static constexpr int block_size = 1; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -983,18 +1128,18 @@ struct IndexGlobal static constexpr int block_size = named_usage::unspecified; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(threadIdx)); + return static_cast(idxNDims.template get_threadIdx()); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return static_cast( - ::RAJA::internal::HipDimHelper::get(blockDim)); + return static_cast(idxNDims.template get_blockDim()); } }; @@ -1006,14 +1151,16 @@ struct IndexGlobal static constexpr int block_size = named_usage::ignored; static constexpr int grid_size = named_usage::ignored; - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(0); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(1); } @@ -1024,17 +1171,19 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index(); + return x_index::template index(idxNDims); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size(); + return x_index::template size(idxNDims); } }; @@ -1043,18 +1192,22 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index() + - x_index::template size() * (y_index::template index()); + return x_index::template index(idxNDims) + + x_index::template size(idxNDims) * + (y_index::template index(idxNDims)); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size() * y_index::template size(); + return x_index::template size(idxNDims) * + y_index::template size(idxNDims); } }; @@ -1063,38 +1216,43 @@ template struct IndexFlatten { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template index() + - x_index::template size() * - (y_index::template index() + - y_index::template size() * - z_index::template index()); + return x_index::template index(idxNDims) + + x_index::template size(idxNDims) * + (y_index::template index(idxNDims) + + y_index::template size(idxNDims) * + z_index::template index(idxNDims)); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return x_index::template size() * y_index::template size() * - z_index::template size(); + return x_index::template size(idxNDims) * + y_index::template size(idxNDims) * + z_index::template size(idxNDims); } }; template struct IndexDivide { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return indexer::template index() / static_cast(divisor); + return indexer::template index(idxNDims) / static_cast(divisor); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { - return RAJA_DIVIDE_CEILING_INT(indexer::template size(), + return RAJA_DIVIDE_CEILING_INT(indexer::template size(idxNDims), static_cast(divisor)); } }; @@ -1102,14 +1260,16 @@ struct IndexDivide template struct IndexModulo { - template - RAJA_DEVICE static inline IdxT index() + template + RAJA_DEVICE static inline IdxT index(IdxNDims const& idxNDims = IdxNDims {}) { - return indexer::template index() % static_cast(divisor); + return indexer::template index(idxNDims) % static_cast(divisor); } - template - RAJA_DEVICE static inline IdxT size() + template + RAJA_DEVICE static inline IdxT size(IdxNDims const& idxNDims = IdxNDims {}) { return static_cast(divisor); }