diff --git a/examples_tests b/examples_tests index efe27fa783..9a380ef63b 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit efe27fa783347e17c43d795395cd32edf73b049d +Subproject commit 9a380ef63b57a19d0108d324f7c49c2e57315e5d diff --git a/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl b/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl index caefc814c5..00ad9f16d8 100644 --- a/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl +++ b/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl @@ -4,7 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/bda/__ptr.hlsl" namespace nbl @@ -66,7 +66,7 @@ struct BdaAccessor : impl::BdaAccessorBase atomicAdd(const uint64_t index, const T value) { bda::__ptr target = ptr + index; - return glsl::atomicAdd(target.template deref().ptr.value, value); + return spirv::atomicIAdd(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } template @@ -74,7 +74,7 @@ struct BdaAccessor : impl::BdaAccessorBase atomicSub(const uint64_t index, const T value) { bda::__ptr target = ptr + index; - return glsl::atomicSub(target.template deref().ptr.value, value); + return spirv::atomicISub(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } bda::__ptr ptr; diff --git a/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl b/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl index f778e2b364..a3fb51ef94 100644 --- a/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl +++ b/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl @@ -9,7 +9,7 @@ #ifndef _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/bda/__ptr.hlsl" diff --git a/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl b/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl index 4b97bbc08f..5eb2457612 100644 --- a/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl +++ b/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl @@ -67,7 +67,7 @@ void main() OutImgAccessor outImgA; outImgA.descIx = pc.outputDescIx; - const uint16_t3 wgID = _static_cast(glsl::gl_WorkGroupID()); + const uint16_t3 wgID = _static_cast(spirv::WorkgroupId); const uint16_t3 baseCoord = pc.perWG.getOutputBaseCoord(wgID); // TODO: If and when someone can be bothered, change the blit api to compile a pipeline per image dimension, maybe it will be faster switch (pc.perWG.imageDim) diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl new file mode 100644 index 0000000000..c1989e7dd0 --- /dev/null +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -0,0 +1,73 @@ +#ifndef _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ +#define _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/generic_shared_data.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +#define NBL_CONCEPT_NAME GenericAtomicAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicStore(index, val)), is_same_v, void)) +); +#undef val +#undef index +#undef accessor +#include + +#define NBL_CONCEPT_NAME DeviceReductionsAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(GenericAtomicAccessor, T, V)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) +); +#undef val +#undef index +#undef accessor +#include + +#define NBL_CONCEPT_NAME GenericAtomicArithmeticAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, uint32_t) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicAdd(index, val)), is_same_v, uint32_t)) +); +#undef val +#undef index +#undef accessor +#include + +} +} +} + +#endif diff --git a/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl b/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl index df785e3e8f..bcba437131 100644 --- a/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl +++ b/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl @@ -4,7 +4,6 @@ #include #include #include -#include #include // TODO: when it will be possible, use this unions wherever they fit: diff --git a/include/nbl/builtin/hlsl/functional.hlsl b/include/nbl/builtin/hlsl/functional.hlsl index 118fe07c63..8c80b6b273 100644 --- a/include/nbl/builtin/hlsl/functional.hlsl +++ b/include/nbl/builtin/hlsl/functional.hlsl @@ -5,7 +5,7 @@ #define _NBL_BUILTIN_HLSL_FUNCTIONAL_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/limits.hlsl" #include "nbl/builtin/hlsl/concepts/vector.hlsl" #include "nbl/builtin/hlsl/array_accessors.hlsl" diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index 5422d0966d..079c62845c 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -16,6 +16,11 @@ namespace nbl { namespace hlsl { +// TODO: rename this namespace to `glsl_compat` +// The functions in this namespace are meant for porting GLSL shaders with minimal changes. +// Many of them are also handy utilities (default scopes and memory semantics, signed/unsigned/float dispatch, 16-bit emulation, +// a C++ path) and get called all over the codebase because they are easier to use than the raw `spirv::` intrinsics. +// Avoid calling them from code that isn't a GLSL port; those utilities need a home outside the `glsl` namespace. namespace glsl { @@ -125,16 +130,6 @@ enable_if_t, T> atomicXor(Ptr_T ptr, T value) { return spirv::atomicXor(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } -/* TODO: @Hazardu struct dispatchers like for `bitfieldExtract` -template -T atomicMin(NBL_REF_ARG(T) ptr, T value) -{ -} -template -T atomicMax(NBL_REF_ARG(T) ptr, T value) -{ -} -*/ // reference form: `ptr` is the variable itself, e.g. `buf[i]`, `bufferPtr.Get().member`, or a groupshared variable with `T` given explicitly template T atomicExchange(NBL_REF_ARG(T) ptr, T value) @@ -160,6 +155,100 @@ enable_if_t, T> atomicCompSwap(Ptr_T ptr, T comparato return spirv::atomicCompareExchange(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, spv::MemorySemanticsMaskNone, value, comparator); } +namespace impl +{ +template +struct atomicMin; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +struct atomicMax; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; +} + +template +T atomicMin(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMin::__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMin(Ptr_T ptr, T value) +{ + return impl::atomicMin::template __call(ptr, value); +} +template +T atomicMax(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMax::__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMax(Ptr_T ptr, T value) +{ + return impl::atomicMax::template __call(ptr, value); +} + /** * GLSL extended math */ @@ -189,7 +278,11 @@ uint32_t gl_InstanceIndex() {return spirv::InstanceIndex;} // TODO: Extemely annoying that HLSL doesn't have references, so we can't transparently alias the variables as `const&` :( uint32_t3 gl_NumWorkGroups() {return spirv::NumWorkGroups;} -// TODO: DXC BUG prevents us from defining this! +// Not a DXC bug: SPIR-V only lets a shader declare its workgroup size, there is nothing to read back. +// `[numthreads]` emits the `LocalSize` execution mode, whose operands are literals that no instruction can reference. +// `LocalSizeId` and the `WorkgroupSize` builtin take ids instead, but those must be constants (`WorkgroupSize` on an input +// variable is invalid), so "reading" them just means reusing the constant that declared the size in the first place. +// Hence the user defines this, returning the same compile-time constant they pass to `[numthreads]`. uint32_t3 gl_WorkGroupSize(); uint32_t3 gl_WorkGroupID() {return spirv::WorkgroupId;} uint32_t3 gl_LocalInvocationID() {return spirv::LocalInvocationId;} @@ -298,6 +391,8 @@ struct bitfieldExtract T bitfieldInsert(T base, T insert, uint32_t offset, uint32_t bits) { @@ -413,6 +508,7 @@ struct notEqual_helper inline vector::Dimension> equal(NBL_CONST_REF_ARG(T) x, NBL_CONST_REF_ARG(T) y) { diff --git a/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl b/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl index 148a61b540..15a015940e 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl @@ -13,6 +13,8 @@ namespace hlsl namespace glsl { +// TODO: these are used as general utilities outside GLSL ports (one name per operation for every type), +// they need a home outside the `glsl` namespace, see the note above `namespace glsl` in glsl_compat/core.hlsl // TODO: @Hazardu this will need touching up in the impl when the `spirv::groupXXX` will change names to a predictable pattern from SPIR-V // TODO: Furthermore you'll need `bitfieldExtract`-like struct dispatcher to choose between int/float add/mul and sint/uint/float min/max template diff --git a/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl b/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl index 2504a34480..20533f0c07 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl @@ -17,6 +17,7 @@ namespace glsl // TODO: Extemely annoying that HLSL doesn't have referencies, so we can't transparently alias the variables as `const&` :( // NOTE: These are not `uint16_t` even though they could be, because IIUtSC they're `uint32_t` in SPIR-V uint32_t gl_SubgroupSize() {return spirv::SubgroupSize;} +// TODO: not a GLSL builtin, move it out of the `glsl` namespace to a better place uint32_t gl_SubgroupSizeLog2() {return firstbithigh(spirv::SubgroupSize);} uint32_t gl_SubgroupInvocationID() {return spirv::SubgroupLocalInvocationId;} diff --git a/include/nbl/builtin/hlsl/memory_accessor.hlsl b/include/nbl/builtin/hlsl/memory_accessor.hlsl index 2194b1e917..f31a9e8db5 100644 --- a/include/nbl/builtin/hlsl/memory_accessor.hlsl +++ b/include/nbl/builtin/hlsl/memory_accessor.hlsl @@ -4,7 +4,6 @@ #ifndef _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" #include "nbl/builtin/hlsl/member_test_macros.hlsl" // weird namespace placing, see the comment where the macro is defined diff --git a/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl b/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl index 577b69fc3c..55e7f78a25 100644 --- a/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl +++ b/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl @@ -1,5 +1,4 @@ #include "nbl/builtin/hlsl/cpp_compat.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" #include "nbl/builtin/hlsl/workgroup/basic.hlsl" #include "nbl/builtin/hlsl/workgroup/arithmetic.hlsl" #include "nbl/builtin/hlsl/device_capabilities_traits.hlsl" diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl new file mode 100644 index 0000000000..3364601c21 --- /dev/null +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -0,0 +1,268 @@ +#ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl" +#include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" + +groupshared bool sIsLocked; + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +namespace impl +{ +template +struct WorkgroupDataProxy +{ + using dtype_t = vector; + + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = uint16_t(VirtualWorkgroupSize / WorkgroupSize); + + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) + { + WorkgroupDataProxy retval; + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf, outputBuf); + retval.workgroupID = workgroupId; + return retval; + } + + template + void get(const IndexType ix, NBL_REF_ARG(AccessType) value) + { + value = preloaded[ix>>WorkgroupSizeLog2]; + } + template + void set(const IndexType ix, const AccessType value) + { + preloaded[ix>>WorkgroupSizeLog2] = value; + } + + void preload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.get(workgroupID * VirtualWorkgroupSize + idx * WorkgroupSize + invocIx, preloaded[idx]); + } + + void workgroupExecutionAndMemoryBarrier() + { + spirv::controlBarrier(spv::ScopeWorkgroup, spv::ScopeWorkgroup, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask); + //glsl::memoryBarrierShared(); implied by the above + } + + DoubleLegacyBdaAccessor accessor; + dtype_t preloaded[PreloadedDataCount]; + uint32_t workgroupID; +}; + +template +struct Scan +{ + using scalar_t = typename BinOp::type_t; + using vector_t = vector; // data accessor needs to be this type + using binop_t = BinOp; + + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Inclusive = 2; // workgroup has summed all preceding groups and added to own sum + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Mask = 3; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; + + // IMPORTANT NOTE: for binary operations scalar precision, we reserve two bits for packing extra flags + // thus, the user will only get 30-bit precision if they use uint32_t + + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); + NBL_CONSTEXPR_STATIC_INLINE uint16_t MaxSpinCount = uint16_t(4u); + + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + if (invocIx == 0u) + { + const uint32_t id = workgroupCounter.atomicAdd(0u, 1u); + scratchAccessor.template set(0u, id); + sIsLocked = true; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + uint16_t workgroupId; + scratchAccessor.template get(0u, workgroupId); + + binop_t binop; + using wg_data_proxy_t = WorkgroupDataProxy; + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); + scalar_t currGroupReduction; + { + wgDataAccessor.preload(); + + // should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + const uint32_t lastInvocIx = spirv::SubgroupSize * spirv::NumSubgroups - 1u; + scalar_t lastElem; + if (invocIx == lastInvocIx) + lastElem = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + + if (Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + else + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + if (invocIx == lastInvocIx) + { + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (Exclusive) + currGroupReduction = binop(currGroupReduction, lastElem); + scratchAccessor.template set(0u, currGroupReduction); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scratchAccessor.template get(0u, currGroupReduction); + } + + if (invocIx == 0u) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; + workgroupReduction.atomicStore(workgroupId, storeVal); + } + + if (workgroupId > 0u) + { + bool locked = sIsLocked; + + scalar_t prevReduction = binop_t::identity; + uint16_t lookbackIx = workgroupId - uint16_t(1u); + + while (locked) + { + // lookback: try to get reduction from previous workgroups + if (invocIx == 0u) + { + uint16_t spinCount = uint16_t(0u); + [loop] + while (spinCount < MaxSpinCount) + { + scalar_t flagPayload; + workgroupReduction.template get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + spinCount = uint16_t(0u); + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicStore(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; + } + else + spinCount++; + } + + // broadcast id and prepare to do reduction ourselves + if (spinCount == MaxSpinCount) + scratchAccessor.template set(0u, lookbackIx); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + // Fallback path: we spun to MaxSpinCount, but no previous workgroup had their global reduction ready (Flag_Inclusive). + // So we try to do reduction for all previous work groups one by one until we find a Flag_Inclusive + if (locked) + { + // do reduction for lookbackIx workgroup + uint16_t fallbackGroupId; + scratchAccessor.template get(0u, fallbackGroupId); + + wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); + fallbackDataAccessor.preload(); + scalar_t fallbackReduction = workgroup2::reduction::template __call(fallbackDataAccessor, scratchAccessor); + + if (invocIx == 0u) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); + const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); + + prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); + if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicStore(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + } + else + lookbackIx--; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + } + } + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + dataAccessor.initAtWorkgroupID(workgroupId); + + scalar_t prevReduction = binop_t::identity; + if (workgroupId) + scratchAccessor.template get(0u, prevReduction); + + NBL_UNROLL + for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) + { + vector_t data = wgDataAccessor.preloaded[idx]; + NBL_UNROLL + for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) + data[i] = binop(prevReduction, data[i]); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + } + } +}; +} + +template) +struct inclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && concepts::accessors::GenericSharedMemoryAccessor && DeviceReductionsAccessor && GenericAtomicArithmeticAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) + { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); + } +}; + +template) +struct exclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && concepts::accessors::GenericSharedMemoryAccessor && DeviceReductionsAccessor && GenericAtomicArithmeticAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) + { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); + } +}; + +} +} +} + +#endif diff --git a/include/nbl/builtin/hlsl/sort/counting.hlsl b/include/nbl/builtin/hlsl/sort/counting.hlsl index 1cd916ccc8..f5ecd827aa 100644 --- a/include/nbl/builtin/hlsl/sort/counting.hlsl +++ b/include/nbl/builtin/hlsl/sort/counting.hlsl @@ -131,7 +131,7 @@ struct counting sdata.workgroupExecutionAndMemoryBarrier(); uint32_t tid = workgroup::SubgroupContiguousIndex(); - const uint32_t shift = glsl::gl_SubgroupSize() * workGroupIndex; + const uint32_t shift = spirv::SubgroupSize * workGroupIndex; for (uint32_t vtid=tid; vtid must be a constant for WorkgroupSize"), see `glsl::gl_WorkGroupSize` //[[vk::ext_builtin_input(spv::BuiltInWorkgroupSize)]] //static const uint32_t3 WorkgroupSize; [[vk::ext_builtin_input(spv::BuiltInWorkgroupId)]] @@ -160,6 +162,9 @@ template [[vk::ext_instruction(spv::OpCopyLogical)]] enable_if_t/* && !is_same_v*/,T> copyLogical(Ptr_U v); +// Atomics still don't detect the address space of `ptr` and narrow down the sync-scope properly, hence the DXC workarounds +// see issue: https://github.com/microsoft/DirectXShaderCompiler/issues/6508 -- same as in glsl_compat/core.hlsl + // Here's the thing with atomics, it's not only the data type that dictates whether you can do an atomic or not. // It's the storage class that has the most effect (shared vs storage vs image) and we can't check that easily // reference form: `ptr` is the variable itself, e.g. a groupshared variable, `buf[i]` or `bufferPtr.Get().member` @@ -296,6 +301,22 @@ template [[vk::ext_instruction(spv::OpAtomicCompareExchange)]] enable_if_t, T> atomicCompareExchange(Ptr_T ptr, uint32_t memoryScope, uint32_t memSemanticsEqual, uint32_t memSemanticsUnequal, T value, T comparator); +template +[[vk::ext_instruction(spv::OpAtomicLoad)]] +T atomicLoad([[vk::ext_reference]] T ptr, uint32_t memoryScope, uint32_t memorySemantics); + +template // DXC Workaround +[[vk::ext_instruction(spv::OpAtomicLoad)]] +enable_if_t, T> atomicLoad(Ptr_T ptr, uint32_t memoryScope, uint32_t memorySemantics); + +template +[[vk::ext_instruction(spv::OpAtomicStore)]] +enable_if_t, void> atomicStore([[vk::ext_reference]] T ptr, uint32_t memoryScope, uint32_t memorySemantics, T value); + +template // DXC Workaround +[[vk::ext_instruction(spv::OpAtomicStore)]] +enable_if_t && is_scalar_v, void> atomicStore(Ptr_T ptr, uint32_t memoryScope, uint32_t memorySemantics, T value); + template __NBL_CAPABILITY_PhysicalStorageBufferAddresses diff --git a/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl b/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl index ca5dd51dad..ea1e7197df 100644 --- a/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl @@ -5,6 +5,7 @@ #define _NBL_BUILTIN_HLSL_SUBGROUP_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ +// TODO: `glsl::subgroupShuffleUp` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" #include "nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl" @@ -94,8 +95,8 @@ struct inclusive_scan static type_t __call(type_t value) { Binop op; - const uint subgroupInvocation = glsl::gl_SubgroupInvocationID(); - const uint halfSubgroupSize = glsl::gl_SubgroupSize() >> 1u; + const uint subgroupInvocation = spirv::SubgroupLocalInvocationId; + const uint halfSubgroupSize = spirv::SubgroupSize >> 1u; type_t rhs = glsl::subgroupShuffleUp(value, 1u); // all invocations must execute the shuffle, even if we don't apply the op() to all of them value = op(value, subgroupInvocation<1u ? Binop::identity:rhs); @@ -121,7 +122,7 @@ struct exclusive_scan // can't risk getting short-circuited, need to store to a var type_t left = glsl::subgroupShuffleUp(value,1); // the first invocation doesn't have anything in its left so we set to the binop's identity value for exlusive scan - return bool(glsl::gl_SubgroupInvocationID()) ? left:Binop::identity; + return bool(spirv::SubgroupLocalInvocationId) ? left:Binop::identity; } }; diff --git a/include/nbl/builtin/hlsl/subgroup/ballot.hlsl b/include/nbl/builtin/hlsl/subgroup/ballot.hlsl index 4c9a817415..196a136891 100644 --- a/include/nbl/builtin/hlsl/subgroup/ballot.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/ballot.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP_BALLOT_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP_BALLOT_INCLUDED_ +// TODO: `glsl::subgroupBallot`, `glsl::subgroupBallotFindMSB`, `glsl::subgroupBroadcast` and `glsl::subgroupBroadcastFirst` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" #include "nbl/builtin/hlsl/subgroup/basic.hlsl" @@ -24,7 +25,7 @@ uint32_t LastSubgroupInvocation() bool ElectLast() { - return glsl::gl_SubgroupInvocationID()==LastSubgroupInvocation(); + return spirv::SubgroupLocalInvocationId==LastSubgroupInvocation(); } template @@ -34,7 +35,7 @@ T BroadcastLast(T value) } uint32_t ElectedSubgroupInvocationID() { - return glsl::subgroupBroadcastFirst(glsl::gl_SubgroupInvocationID()); + return glsl::subgroupBroadcastFirst(spirv::SubgroupLocalInvocationId); } } diff --git a/include/nbl/builtin/hlsl/subgroup/basic.hlsl b/include/nbl/builtin/hlsl/subgroup/basic.hlsl index b39c176e8e..de8d12d6c3 100644 --- a/include/nbl/builtin/hlsl/subgroup/basic.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/basic.hlsl @@ -4,8 +4,6 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP_BASIC_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP_BASIC_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" - namespace nbl { namespace hlsl diff --git a/include/nbl/builtin/hlsl/subgroup/fft.hlsl b/include/nbl/builtin/hlsl/subgroup/fft.hlsl index f12f3e72e1..7c306f91df 100644 --- a/include/nbl/builtin/hlsl/subgroup/fft.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/fft.hlsl @@ -2,7 +2,8 @@ #define _NBL_BUILTIN_HLSL_SUBGROUP_FFT_INCLUDED_ #include "nbl/builtin/hlsl/fft/common.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/subgroup_basic.hlsl" +// TODO: `glsl::subgroupShuffleXor` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" namespace nbl @@ -26,7 +27,7 @@ struct FFT { static void FFT_loop(uint32_t stride, NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const bool topHalf = bool(glsl::gl_SubgroupInvocationID() & stride); + const bool topHalf = bool(spirv::SubgroupLocalInvocationId & stride); const vector toTrade = topHalf ? vector (lo.real(), lo.imag()) : vector (hi.real(), hi.imag()); const vector exchanged = glsl::subgroupShuffleXor< vector > (toTrade, stride); if (topHalf) @@ -40,15 +41,15 @@ struct FFT hi.imag(exchanged.y); } // Get twiddle with k = subgroupInvocation mod stride, halfN = stride - fft::DIF::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID() & (stride - 1), stride), lo, hi); + fft::DIF::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId & (stride - 1), stride), lo, hi); } static void __call(NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const uint32_t subgroupSize = glsl::gl_SubgroupSize(); //This is N/2 + const uint32_t subgroupSize = spirv::SubgroupSize; //This is N/2 // special first iteration - fft::DIF::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID(), subgroupSize), lo, hi); + fft::DIF::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId, subgroupSize), lo, hi); // Decimation in Frequency [unroll] @@ -66,9 +67,9 @@ struct FFT static void FFT_loop(uint32_t stride, NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { // Get twiddle with k = subgroupInvocation mod stride, halfN = stride - fft::DIT::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID() & (stride - 1), stride), lo, hi); + fft::DIT::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId & (stride - 1), stride), lo, hi); - const bool topHalf = bool(glsl::gl_SubgroupInvocationID() & stride); + const bool topHalf = bool(spirv::SubgroupLocalInvocationId & stride); const vector toTrade = topHalf ? vector (lo.real(), lo.imag()) : vector (hi.real(), hi.imag()); const vector exchanged = glsl::subgroupShuffleXor< vector > (toTrade, stride); if (topHalf) @@ -85,7 +86,7 @@ struct FFT static void __call(NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const uint32_t subgroupSize = glsl::gl_SubgroupSize(); //This is N/2 + const uint32_t subgroupSize = spirv::SubgroupSize; //This is N/2 const uint32_t doubleSubgroupSize = subgroupSize << 1; //This is N // Decimation in Time @@ -94,7 +95,7 @@ struct FFT FFT_loop(stride, lo, hi); // special last iteration - fft::DIT::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID(), subgroupSize), lo, hi); + fft::DIT::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId, subgroupSize), lo, hi); divides_assign< complex_t > divAss; divAss(lo, Scalar(doubleSubgroupSize)); divAss(hi, Scalar(doubleSubgroupSize)); diff --git a/include/nbl/builtin/hlsl/subgroup2/README.md b/include/nbl/builtin/hlsl/subgroup2/README.md new file mode 100644 index 0000000000..d46bc03d81 --- /dev/null +++ b/include/nbl/builtin/hlsl/subgroup2/README.md @@ -0,0 +1,130 @@ +# subgroup2 arithmetic + +Not every device supports subgroup arithmetic (`VK_SUBGROUP_FEATURE_ARITHMETIC_BIT`). So `subgroup2::reduction`, `subgroup2::inclusive_scan` and `subgroup2::exclusive_scan` either use the native subgroup instructions or emulate them with subgroup shuffles. The choice is made at compile time from `device_capabilities::shaderSubgroupArithmetic`, which you set from the device's limits on the host. + +Both paths combine values across one subgroup, and the rest of the configuration comes from the `ArithmeticParams` you pass in. + +- [ArithmeticParams](#arithmeticparams) +- [What ItemsPerInvocation is](#what-itemsperinvocation-is) +- [How the specializations are chosen](#how-the-specializations-are-chosen) +- [Seeing it in SPIR-V](#seeing-it-in-spir-v) + +## ArithmeticParams + +```cpp +using config_t = subgroup2::Configuration<5>; // SubgroupSize 32 +using params_t = subgroup2::ArithmeticParams, 1, device_capabilities>; +subgroup2::inclusive_scan scan; +const params_t::type_t result = scan(value); +``` + +| `ArithmeticParams` member | Comes from | +|---|---| +| `config_t` | `Configuration`. Must match the subgroup size the pipeline runs with. | +| `binop_t`, `scalar_t` | the binop, e.g. `plus`, and its `type_t` | +| `type_t` | `vector`: each invocation holds `ItemsPerInvocation` consecutive elements | +| `UseNativeIntrinsics` | `device_capabilities::shaderSubgroupArithmetic`, `false` when the member is missing or `device_capabilities` is `void` | + +## What ItemsPerInvocation is + +**Each thread holds N consecutive elements in one `vector`, instead of one element. That's all it means.** (N is `ItemsPerInvocation`.) + +With a subgroup of 32 and `ItemsPerInvocation = 2`, one subgroup covers 32 × 2 = 64 elements, and lane `k` holds elements `2k` and `2k+1`: + +| lane | holds | element indices | +|---|---|---| +| 0 | `(e0, e1)` | 0, 1 | +| 1 | `(e2, e3)` | 2, 3 | +| ... | ... | ... | +| 31 | `(e62, e63)` | 62, 63 | + +The subgroup still has 32 lanes, and an operation still needs only one cross-lane step. The extra items are combined serially inside each invocation. + +For example, take an inclusive `plus` scan of 64 ones. The steps are the numbered comments in the [primary `inclusive_scan`](arithmetic_portability_impl.hlsl#L39-L72): + +| | lane 0 | lane 1 | lane 2 | ... | lane 31 | +|---|---|---|---|---|---| +| holds | (1, 1) | (1, 1) | (1, 1) | ... | (1, 1) | +| (1) own items scanned | (1, **2**) | (1, **2**) | (1, **2**) | ... | (1, **2**) | +| (2) subgroup exclusive scan of the totals | 0 | 2 | 4 | ... | 62 | +| (3) combined with every item | (1, 2) | (3, 4) | (5, 6) | ... | (63, 64) | + +`workgroup2::ArithmeticConfiguration`'s `ItemsPerInvocation_0` is this same value. It's the vector type workgroup data accessors read and write. + +## How the specializations are chosen + +The public structs in `arithmetic_portability.hlsl` inherit from `impl::`, defined in `arithmetic_portability_impl.hlsl`. **`ItemsPerInvocation` is what selects the primary template, not native support:** + +| `ItemsPerInvocation` | native | Selected | What it does | +|---|---|---|---| +| not 1 | either | primary template | Combines the invocation's own items serially, then makes one `ItemsPerInvocation = 1` call (the exclusive scan for both scans, the reduction for `reduction`) on each invocation's total. See [What ItemsPerInvocation is](#what-itemsperinvocation-is). | +| 1 | `true` | partial specialization on `BinOp`, written out by the `SPECIALIZE_ALL` macro for `bit_and`, `bit_or`, `bit_xor`, `plus`, `multiplies`, `minimum`, `maximum` | one `glsl::subgroup{Inclusive,Exclusive,}{And,Or,Xor,Add,Mul,Min,Max}` call | +| 1 | `false` | partial specialization on ``, for any binop | Emulation with `SizeLog2` shuffle steps. Inclusive scan: Hillis-Steele with `subgroupShuffleUp`. Exclusive scan: shift up by one lane, then inclusive. Reduction: `subgroupShuffleXor` butterfly. | + +So native support only decides which of the two `ItemsPerInvocation = 1` versions does the cross-lane step, whether it's called directly or from inside the primary template. + +A binop without a `SPECIALIZE_ALL` line has no `<..., 1, true>` specialization. With native on, even `ItemsPerInvocation = 1` falls back to the primary template, which then calls itself, and DXC rejects it with `recursive functions are not allowed`. Use custom binops with `shaderSubgroupArithmetic = false`. + +## Seeing it in SPIR-V + +The shader below compiles one inclusive `plus` scan over a subgroup of 32. Change the three macros to pick a different specialization. + +```cpp +#include "nbl/builtin/hlsl/functional.hlsl" +#include "nbl/builtin/hlsl/workgroup/basic.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_params.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_portability.hlsl" + +// change these to pick a different specialization +#ifndef SUBGROUP_SIZE_LOG2 +#define SUBGROUP_SIZE_LOG2 5 // subgroup of 32 +#endif +#ifndef ITEMS_PER_INVOCATION +#define ITEMS_PER_INVOCATION 1 +#endif +#ifndef NATIVE +#define NATIVE 1 +#endif + +struct device_capabilities +{ + NBL_CONSTEXPR_STATIC_INLINE bool shaderSubgroupArithmetic = bool(NATIVE); +}; + +using namespace nbl::hlsl; +using config_t = subgroup2::Configuration; +using params_t = subgroup2::ArithmeticParams, ITEMS_PER_INVOCATION, device_capabilities>; + +[[vk::binding(0,0)]] RWStructuredBuffer data; + +[numthreads(64,1,1)] +void main() +{ + const uint32_t ix = workgroup::SubgroupContiguousIndex(); + subgroup2::inclusive_scan scan; + data[ix] = scan(data[ix]); +} +``` + +``` +nsc -spirv -HV 202x -T cs_6_7 -E main -fspv-target-env=vulkan1.3 -enable-16bit-types -DNATIVE=1 -DITEMS_PER_INVOCATION=1 -Fc scan.spv scan.hlsl +``` + +On Compiler Explorer (the release `nsc`, not necessarily your branch): + +| Variant | Link | +|---|---| +| native, 1 item | https://godbolt.devsh.eu/z/Waoao1 | +| emulated, 1 item | https://godbolt.devsh.eu/z/Yexhc4 | +| native, 4 items | https://godbolt.devsh.eu/z/z5rbno | +| emulated, 4 items | https://godbolt.devsh.eu/z/Y67zWK | +| all four side by side | https://godbolt.devsh.eu/z/abM995 | + +What each variant compiles to, with SubgroupSize 32: + +| Variant | Subgroup instructions | `OpIAdd` | +|---|---|---| +| native, 1 item | 1 `OpGroupNonUniformIAdd InclusiveScan` | 1: the `SubgroupContiguousIndex` | +| emulated, 1 item | 5 × (`OpControlBarrier`, `OpGroupNonUniformShuffleUp`, `OpSelect`), one per shuffle step | 6: 5 steps + index | +| native, 4 items | 1 `OpGroupNonUniformIAdd ExclusiveScan` (the primary template asks for the exclusive scan) | 8: 3 to scan the 4 items + 4 to add the exclusive result + index | +| emulated, 4 items | 6 × (`OpControlBarrier`, `OpGroupNonUniformShuffleUp`, `OpSelect`): 1 to shift up a lane + 5 inclusive steps | 13: 3 + 5 + 4 + index | diff --git a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl index 18f20c3cfc..b909db02f7 100644 --- a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP2_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP2_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ +// TODO: `glsl::subgroupShuffleUp` and `glsl::subgroupShuffleXor` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" #include "nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl" @@ -35,6 +36,9 @@ template struct reduction; +// Primary templates, used when ItemsPerInvocation != 1: each invocation holds ItemsPerInvocation consecutive elements in a vector. +// They combine the invocation's own items serially, so a single `ItemsPerInvocation = 1` subgroup op (native or emulated, +// specialized below) is all that crosses lanes. // BinOp needed to specialize native template struct inclusive_scan @@ -49,14 +53,17 @@ struct inclusive_scan { binop_t binop; type_t retval; + // (1) inclusive scan of this invocation's own items retval[0] = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i-1], value[i]); + // (2) one subgroup exclusive scan of each invocation's total: everything held by the lanes before this one exclusive_scan_op_t op; scalar_t exclusive = op(retval[ItemsPerInvocation-1]); + // (3) combine that with every item [unroll] for (uint32_t i = 0; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i], exclusive); @@ -76,14 +83,17 @@ struct exclusive_scan { binop_t binop; type_t retval; + // (1) inclusive scan of this invocation's own items retval[0] = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i-1], value[i]); + // (2) one subgroup exclusive scan of each invocation's total: everything held by the lanes before this one exclusive_scan_op_t op; scalar_t exclusive = op(retval[ItemsPerInvocation-1]); + // (3) shift the items up by one to make them exclusive, combining each with that; item 0 gets it as is [unroll] for (uint32_t i = ItemsPerInvocation-1; i > 0; i--) retval[i] = binop(exclusive,retval[i-1]); @@ -104,16 +114,21 @@ struct reduction { binop_t binop; op_t op; + // (1) reduce this invocation's own items scalar_t retval = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval = binop(retval, value[i]); + // (2) one subgroup reduction of each invocation's total return op(retval); } }; // specs for N=1 uses subgroup funcs +// TODO: drop the weird `glsl::` dependency, call `spirv::group*` directly with `spv::ScopeSubgroup` and the `GroupOperation`, or a utility outside `glsl`. +// `plus`, `multiplies` and the bitwise ops map 1:1; `minimum`/`maximum` need signed/unsigned/float dispatch (`spirv::groupSMin/UMin/FMin`), +// either as 3 constrained specializations each (primaries need `NBL_STRUCT_CONSTRAINABLE`) or a shared dispatch helper also used by `glsl::subgroupMin/Max`. // specialize native #define SPECIALIZE(NAME,BINOP,SUBGROUP_OP) template struct NAME,1,true> \ { \ @@ -148,7 +163,7 @@ struct inclusive_scan_impl { inclusive_scan_impl retval; retval.value = _value; - retval.subgroupInvocation = glsl::gl_SubgroupInvocationID(); + retval.subgroupInvocation = spirv::SubgroupLocalInvocationId; return retval; } @@ -200,7 +215,7 @@ struct exclusive_scan // sync up each subgroup invocation so it runs in lockstep spirv::controlBarrier(spv::ScopeSubgroup, spv::ScopeSubgroup, spv::MemorySemanticsMaskNone); - scalar_t left = hlsl::mix(binop_t::identity, glsl::subgroupShuffleUp(value,1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left = hlsl::mix(binop_t::identity, glsl::subgroupShuffleUp(value,1), bool(spirv::SubgroupLocalInvocationId)); return inclusive_scan::__call(left); } }; diff --git a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl index 3b511126b4..74e2bfa4fb 100644 --- a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP2_BALLOT_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP2_BALLOT_INCLUDED_ +// TODO: `glsl::subgroupBallot` and `glsl::subgroupBallotFindMSB` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" namespace nbl @@ -17,19 +18,20 @@ template uint32_t LastSubgroupInvocation() { if (AssumeAllActive) - return glsl::gl_SubgroupSize()-1; + return spirv::SubgroupSize-1; else return glsl::subgroupBallotFindMSB(glsl::subgroupBallot(true)); } bool ElectLast() { - return glsl::gl_SubgroupInvocationID()==LastSubgroupInvocation(); + return spirv::SubgroupLocalInvocationId==LastSubgroupInvocation(); } template struct Configuration { + // smallest type holding one bit per subgroup invocation, for ballot-style masks: `uint32_t1` up to a SubgroupSize of 32, `uint32_t2` for 64, `uint32_t4` for 128 using mask_t = conditional_t, uint32_t4>; NBL_CONSTEXPR_STATIC_INLINE uint16_t SizeLog2 = uint16_t(SubgroupSizeLog2); diff --git a/include/nbl/builtin/hlsl/surface_transform.h b/include/nbl/builtin/hlsl/surface_transform.h index 0c9b54ce5a..88f4f9ddc2 100644 --- a/include/nbl/builtin/hlsl/surface_transform.h +++ b/include/nbl/builtin/hlsl/surface_transform.h @@ -5,7 +5,6 @@ #define _NBL_BUILTIN_HLSL_SURFACE_TRANSFORM_INCLUDED_ #include #include -#include namespace nbl { diff --git a/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl b/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl index f8c5c75cb0..a57788acfd 100644 --- a/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl @@ -115,7 +115,7 @@ uint16_t ballotScanBitCount(NBL_REF_ARG(BallotAccessor) ballotAccessor, NBL_REF_ arithmeticAccessor.set(subgroupIndex,count); arithmeticAccessor.workgroupExecutionAndMemoryBarrier(); arithmeticAccessor.get(bitfieldIndex, count); - return uint16_t(countbits(localBitfield&(Exclusive ? glsl::gl_SubgroupLtMask():glsl::gl_SubgroupLeMask())[getDWORD(uint16_t(glsl::gl_SubgroupInvocationID()))])+count); + return uint16_t(countbits(localBitfield&(Exclusive ? spirv::BuiltInSubgroupLtMask : spirv::BuiltInSubgroupLeMask)[getDWORD(uint16_t(spirv::SubgroupLocalInvocationId))])+count); } } diff --git a/include/nbl/builtin/hlsl/workgroup/ballot.hlsl b/include/nbl/builtin/hlsl/workgroup/ballot.hlsl index bcfdde8c4b..c6cecd0e82 100644 --- a/include/nbl/builtin/hlsl/workgroup/ballot.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/ballot.hlsl @@ -8,6 +8,8 @@ #include "nbl/builtin/hlsl/workgroup/basic.hlsl" #include "nbl/builtin/hlsl/subgroup/arithmetic_portability.hlsl" +// TODO: `glsl::subgroupElect` and `glsl::subgroupBallot` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace + namespace nbl { namespace hlsl @@ -43,7 +45,7 @@ struct ballot if (glsl::subgroupElect()) { const uint16_t SubgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); const uint16_t shift = (subgroupID<>(5-SubgroupSizeLog2),subgroupBallot[0]< static void __call(const uint32_t4 subgroupBallot, const uint16_t subgroupInvocation, NBL_REF_ARG(Accessor) accessor) { const uint16_t SubgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); const uint16_t destIx = subgroupID<<(SubgroupSizeLog2-5); const uint16_t UsefulComponents = uint16_t(0x1u)<<(SubgroupSizeLog2-5); @@ -89,7 +91,7 @@ template void ballot(const bool value, NBL_REF_ARG(Accessor) accessor) { const uint32_t4 bitfield = glsl::subgroupBallot(value); - const uint16_t subgroupInvocation = uint16_t(glsl::gl_SubgroupInvocationID()); + const uint16_t subgroupInvocation = uint16_t(spirv::SubgroupLocalInvocationId); if (glsl::gl_SubgroupSizeLog2()<4) impl::ballot::template __call(bitfield,subgroupInvocation,accessor); diff --git a/include/nbl/builtin/hlsl/workgroup/basic.hlsl b/include/nbl/builtin/hlsl/workgroup/basic.hlsl index 3467b46407..79af5f74ee 100644 --- a/include/nbl/builtin/hlsl/workgroup/basic.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/basic.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_WORKGROUP_BASIC_INCLUDED_ #define _NBL_BUILTIN_HLSL_WORKGROUP_BASIC_INCLUDED_ +// TODO: `glsl::subgroupBroadcastFirst` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" //! all functions must be called in uniform control flow (all workgroup invocations active) @@ -22,17 +23,20 @@ uint16_t Volume() const uint16_t3 dims = uint16_t3(glsl::gl_WorkGroupSize()); return dims.x*dims.y*dims.z; } - + +// Use this instead of `LocalInvocationIndex`: that doesn't guarantee indices 0..31 come from the first subgroup (with a SubgroupSize of 32), this function does. +// Vulkan defines no relationship between `SubgroupLocalInvocationId` and `LocalInvocationIndex`, and suggests this formula instead, +// valid only with full subgroups: https://docs.vulkan.org/refpages/latest/refpages/source/SubgroupLocalInvocationId.html uint16_t SubgroupContiguousIndex() { - const uint16_t retval = (uint16_t(glsl::gl_SubgroupID())<(otherElementIndex, workgroup::SubgroupContiguousIndex(), 0, uint32_t(WorkgroupSizeLog2)); + const uint32_t mirrorGlobalIndex = spirv::bitFieldInsert(otherElementIndex, workgroup::SubgroupContiguousIndex(), 0, uint32_t(WorkgroupSizeLog2)); const uint32_t otherThreadID = otherElementIndex & (WorkgroupSize - 1); const NablaMirrorGlobalInfo info = { otherThreadID, mirrorGlobalIndex }; return info; @@ -339,7 +339,7 @@ struct FFT, device accessor.get(hiIx, hi); // If for some reason you're running a small FFT, skip all the bigger-than-subgroup steps - if (WorkgroupSize > glsl::gl_SubgroupSize()) + if (WorkgroupSize > spirv::SubgroupSize) { // Set up the memory adaptor using adaptor_t = accessor_adaptors::StructureOfArrays; @@ -351,14 +351,14 @@ struct FFT, device // Run bigger steps until Subgroup-sized [unroll] - for (uint32_t stride = WorkgroupSize >> 1; stride > glsl::gl_SubgroupSize(); stride >>= 1) + for (uint32_t stride = WorkgroupSize >> 1; stride > spirv::SubgroupSize; stride >>= 1) { FFT_loop< adaptor_t >(stride, lo, hi, threadID, sharedmemAdaptor); sharedmemAdaptor.workgroupExecutionAndMemoryBarrier(); } // special last workgroup-shuffle - fft::impl::exchangeValues::__call(lo, hi, threadID, glsl::gl_SubgroupSize(), sharedmemAdaptor); + fft::impl::exchangeValues::__call(lo, hi, threadID, spirv::SubgroupSize, sharedmemAdaptor); // Remember to update the accessor's state sharedmemAccessor = sharedmemAdaptor.accessor; @@ -408,7 +408,7 @@ struct FFT, device_ subgroup::FFT::__call(lo, hi); // If for some reason you're running a small FFT, skip all the bigger-than-subgroup steps - if (WorkgroupSize > glsl::gl_SubgroupSize()) + if (WorkgroupSize > spirv::SubgroupSize) { // Set up the memory adaptor using adaptor_t = accessor_adaptors::StructureOfArrays; @@ -416,11 +416,11 @@ struct FFT, device_ sharedmemAdaptor.accessor = sharedmemAccessor; // special first workgroup-shuffle - fft::impl::exchangeValues::__call(lo, hi, threadID, glsl::gl_SubgroupSize(), sharedmemAdaptor); + fft::impl::exchangeValues::__call(lo, hi, threadID, spirv::SubgroupSize, sharedmemAdaptor); // The bigger steps [unroll] - for (uint32_t stride = glsl::gl_SubgroupSize() << 1; stride < WorkgroupSize; stride <<= 1) + for (uint32_t stride = spirv::SubgroupSize << 1; stride < WorkgroupSize; stride <<= 1) { // Order of waiting for shared mem writes is also reversed here, since the shuffle came earlier sharedmemAdaptor.workgroupExecutionAndMemoryBarrier(); @@ -430,8 +430,8 @@ struct FFT, device_ // special last iteration hlsl::fft::DIT::radix2(hlsl::fft::twiddle(threadID, WorkgroupSize), lo, hi); divides_assign< complex_t > divAss; - divAss(lo, Scalar(WorkgroupSize / glsl::gl_SubgroupSize())); - divAss(hi, Scalar(WorkgroupSize / glsl::gl_SubgroupSize())); + divAss(lo, Scalar(WorkgroupSize / spirv::SubgroupSize)); + divAss(hi, Scalar(WorkgroupSize / spirv::SubgroupSize)); // Remember to update the accessor's state sharedmemAccessor = sharedmemAdaptor.accessor; diff --git a/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl b/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl index b52d083733..8a3bb11241 100644 --- a/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl @@ -6,7 +6,8 @@ #include "nbl/builtin/hlsl/cpp_compat.hlsl" #include "nbl/builtin/hlsl/workgroup/broadcast.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +// TODO: `glsl::subgroupElect` and `glsl::subgroupShuffleUp` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace +#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" #include "nbl/builtin/hlsl/subgroup/arithmetic_portability.hlsl" namespace nbl @@ -27,7 +28,7 @@ struct reduce void __call(NBL_CONST_REF_ARG(type_t) value, NBL_REF_ARG(Accessor) scratchAccessor) { const uint16_t lastInvocation = ItemCount-_static_cast(1); - const uint16_t subgroupMask = _static_cast(glsl::gl_SubgroupSize()-1u); + const uint16_t subgroupMask = _static_cast(spirv::SubgroupSize-1u); subgroup::inclusive_scan subgroupOp; @@ -39,7 +40,7 @@ struct reduce type_t scan = firstLevelScan; // could use ElectLast() but we can optimize for full workgroups here - const bool isLastSubgroupInvocation = uint16_t(glsl::gl_SubgroupInvocationID())==subgroupMask; + const bool isLastSubgroupInvocation = uint16_t(spirv::SubgroupLocalInvocationId)==subgroupMask; // Since we are scanning the RESULT of the initial scan (which paired one input per subgroup invocation) // every group of gl_SubgroupSz invocations has been coallesced into 1 result value. This means that the results of @@ -47,7 +48,7 @@ struct reduce // Consequently, those first gl_SubgroupSz^2 invocations will store their results on gl_SubgroupSz scratch slots // and the next level will follow the same + the previous as an `offset`. - const uint16_t loadStoreIndexDiff = scanLoadIndex-_static_cast(glsl::gl_SubgroupID()); + const uint16_t loadStoreIndexDiff = scanLoadIndex-_static_cast(spirv::SubgroupId); // to cancel out the index shift on the first iteration if (lastInvocationInLevel>subgroupMask) @@ -91,13 +92,13 @@ struct scan// : reduce https://github.com/microsoft/DirectXShad { __base.template __call(value,scratchAccessor); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); // abuse integer wraparound to map 0 to 0xffffu const uint16_t prevSubgroupID = subgroupID-_static_cast(1); // important check to prevent weird `firstbithigh` overlflows const uint16_t lastInvocation = ItemCount-_static_cast(1); - if(lastInvocation>=uint16_t(glsl::gl_SubgroupSize())) + if(lastInvocation>=uint16_t(spirv::SubgroupSize)) { const uint16_t subgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); // different than Upsweep cause we need to translate high level inclusive scans into exclusive on the fly, so we get the value of the subgroup behind our own in each level diff --git a/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl b/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl index 22c93ce193..3cc3cfc1d1 100644 --- a/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl +++ b/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl @@ -79,7 +79,7 @@ struct ArithmeticConfiguration #ifdef __HLSL_VERSION static bool electLast() { - return glsl::gl_SubgroupInvocationID()==SubgroupSize-1; + return spirv::SubgroupLocalInvocationId==SubgroupSize-1; } #endif diff --git a/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl b/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl index 613ba0ddca..f7a45d4e2f 100644 --- a/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl +++ b/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl @@ -5,13 +5,15 @@ #define _NBL_BUILTIN_HLSL_WORKGROUP2_SHARED_SCAN_INCLUDED_ #include "nbl/builtin/hlsl/workgroup/broadcast.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/subgroup_basic.hlsl" #include "nbl/builtin/hlsl/subgroup2/ballot.hlsl" #include "nbl/builtin/hlsl/subgroup2/arithmetic_params.hlsl" #include "nbl/builtin/hlsl/subgroup2/arithmetic_portability.hlsl" #include "nbl/builtin/hlsl/mpl.hlsl" #include "nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl" +// TODO: `glsl::subgroupShuffleUp` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace + namespace nbl { namespace hlsl @@ -44,7 +46,7 @@ struct reduce subgroup2::reduction reduction; vector_t value; - dataAccessor.template get(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template get(uint16_t(spirv::SubgroupLocalInvocationId), value); return reduction(value); } }; @@ -63,7 +65,7 @@ struct scan using params_t = subgroup2::ArithmeticParams; vector_t value; - dataAccessor.template get(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template get(uint16_t(spirv::SubgroupLocalInvocationId), value); if (Exclusive) { subgroup2::exclusive_scan excl_scan; @@ -74,7 +76,7 @@ struct scan subgroup2::inclusive_scan incl_scan; value = incl_scan(value); } - dataAccessor.template set(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template set(uint16_t(spirv::SubgroupLocalInvocationId), value); } }; @@ -102,7 +104,7 @@ struct reduce_level0 scan_local = reduction0(scan_local); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId), idx); scratchAccessor.template set(bankedIndex, scan_local[Config::ItemsPerInvocation_0-1]); // set last element of subgroup scan (reduction) to level 1 scan } } @@ -134,7 +136,7 @@ struct scan_level0 dataAccessor.template set(idx * Config::WorkgroupSize + virtualInvocationIndex, value); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId), idx); scratchAccessor.template set(bankedIndex, value[Config::ItemsPerInvocation_0-1]); // set last element of subgroup scan (reduction) to level 1 scan } } @@ -162,7 +164,7 @@ struct reduce const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::reduction reduction1; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv1_t lv1_val; [unroll] @@ -200,7 +202,7 @@ struct scan const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::inclusive_scan inclusiveScan1; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv1_t lv1_val; [unroll] @@ -220,13 +222,13 @@ struct scan vector_lv0_t value; dataAccessor.template get(idx * Config::WorkgroupSize + virtualInvocationIndex, value); - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()-1u), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId-1u), idx); scalar_t left = BinOp::identity; - if (idx != 0 || glsl::gl_SubgroupID() != 0) + if (idx != 0 || spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex,left); if (Exclusive) { - scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(spirv::SubgroupLocalInvocationId)); [unroll] for (uint16_t i = Config::ItemsPerInvocation_0-1; i > 0; i--) value[i] = binop(left, value[i-1]); @@ -265,7 +267,7 @@ struct reduce const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::reduction reduction1; - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -274,7 +276,7 @@ struct reduce lv1_val = reduction1(lv1_val); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(glsl::gl_SubgroupID())); + const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(spirv::SubgroupId)); scratchAccessor.template set(bankedIndex, lv1_val[Config::ItemsPerInvocation_1-1]); } } @@ -282,7 +284,7 @@ struct reduce // level 2 scan subgroup2::reduction reduction2; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv2_t lv2_val; [unroll] @@ -321,7 +323,7 @@ struct scan const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::inclusive_scan inclusiveScan1; - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -336,7 +338,7 @@ struct scan // level 2 scan subgroup2::inclusive_scan inclusiveScan2; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { const uint16_t lastChannel = Config::ItemsPerInvocation_1 - uint16_t(1u); vector_lv2_t lv2_val; @@ -355,7 +357,7 @@ struct scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); // combine with level 1 - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -363,8 +365,8 @@ struct scan scratchAccessor.template get(Config::template sharedLoadIndex<1>(invocationIndex, i), lv1_val[i]); scalar_t lv2_scan = BinOp::identity; - const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(glsl::gl_SubgroupID()-1u)); - if (glsl::gl_SubgroupID() != 0) + const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(spirv::SubgroupId-1u)); + if (spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex, lv2_scan); [unroll] @@ -380,13 +382,13 @@ struct scan vector_lv0_t value; dataAccessor.template get(idx * Config::WorkgroupSize + virtualInvocationIndex, value); - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()-1u), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId-1u), idx); scalar_t left = BinOp::identity; - if (idx != 0 || glsl::gl_SubgroupID() != 0) + if (idx != 0 || spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex,left); if (Exclusive) { - scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(spirv::SubgroupLocalInvocationId)); [unroll] for (uint16_t i = Config::ItemsPerInvocation_0-1; i > 0; i--) value[i] = binop(left, value[i-1]); diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index 8701aaee9b..9175eecc2b 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -361,6 +361,8 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/items_per_inv LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/arithmetic_config_def.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/arithmetic.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/shared_scan.hlsl") +#device scan +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/scan/chained_scan.hlsl") #Extensions LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/SVertexAttributes.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/default.vert.hlsl") @@ -384,6 +386,7 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/storable_i LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/generic_shared_data.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/fft.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/workgroup_arithmetic.hlsl") +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/device_arithmetic.hlsl") #tgmath LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath/impl.hlsl")