From 007f23ed3b3ee5f9df33496a05131fe538dc8f93 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 14 Jul 2026 16:57:59 +0700 Subject: [PATCH 01/31] chained scan initial commit w rough outline --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 include/nbl/builtin/hlsl/scan/chained_scan.hlsl diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl new file mode 100644 index 0000000000..b24278fdb8 --- /dev/null +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -0,0 +1,119 @@ +#ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ + +#include "nbl/builtin/hlsl/workgroup2/shared_scan.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +namespace impl +{ +template +struct scan +{ + using scalar_t = typename BinOp::type_t; + using vector_t = vector; // data accessor needs to be this type + + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Inclusive = 2; // workgroup has summed all preceding groups and added to own sum + NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Mask = 3; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; + + // TODO: I think we need separate DataAccessor classes between device-wide and workgroup scan + + // TODO: double check op, it's all plus right now + + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction + { + // TODO: what to do with config? + + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + const uint16_t workgroupId = glsl::gl_WorkGroupID(); + + // TODO: workgroup scan here, don't want to deal with that now + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + const scalar_t currGroupReduction = // TODO get last elem of curr workgroup; + if (!invocIx) + { + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; + workgroupReduction.atomicExchange(workgroupId, storeVal); + } + + // lookback + if (workgroupId && !invocIx) + { + scalar_t prevReduction = 0u; + uint16_t lookbackIx = workgroupId - uint16_t(1u); + + while (true) // TODO: check if lookbackIx < 0? + { + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + prevReduction += flagPayload >> Flag_Shift; + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t groupReduction = // TODO get last elem of workgroup index lookbackIx; + const scalar_t storeVal = Flag_Inclusive | ((prevReduction + groupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.set(0u, prevReduction); + break; + } + else + lookbackIx--; + } + else + lookbackIx--; + } + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scalar_t prevReduction; + scratchAccessor.get(0u, prevReduction); + prevReduction += currGroupReduction; + + // TODO: propagate correctly when data accessor done + dataAccessor[invocIx] += prevReduction; + } +}; +} + +template) +struct inclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + { + impl::scan fn; + fn.template __call(dataAccessor, scratchAccessor); + } +}; + +template) +struct exclusive_scan +{ + using scalar_t = typename BinOp::type_t; + + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + { + } +}; + +} +} +} + +#endif From 9604ca6005438f9d45b5a22db24474814051d069 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 15 Jul 2026 16:21:13 +0700 Subject: [PATCH 02/31] implement proper read/write with separate data accessor for workgroup --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 104 +++++++++++++++--- 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index b24278fdb8..3903816c17 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -12,8 +12,60 @@ namespace scan namespace impl { -template -struct scan +template +struct WorkgroupDataProxy +{ + using dtype_t = vector; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; + + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf) + { + WorkgroupDataProxy retval; + const uint32_t workgroupOffset = glsl::gl_WorkGroupID().x * VirtualWorkgroupSize * sizeof(dtype_t); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); + return retval; + } + + template + void get(const IndexType ix, NBL_REF_ARG(AccessType) value) + { + value = preloaded[ix>>WorkgroupSizeLog2]; + } + template + void set(const IndexType ix, const AccessType value) + { + preloaded[ix>>WorkgroupSizeLog2] = value; + } + + void preload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.get(idx * WorkgroupSize + invocIx, preloaded[idx]); + } + void unload() + { + const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); + NBL_UNROLL + for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) + accessor.set(idx * WorkgroupSize + invocIx, preloaded[idx]); + } + + void workgroupExecutionAndMemoryBarrier() + { + glsl::barrier(); + //glsl::memoryBarrierShared(); implied by the above + } + + DoubleLegacyBdaAccessor accessor; + dtype_t preloaded[PreloadedDataCount]; +}; + +template // TODO: Config is same as workgroup2 stuff? +struct Scan { using scalar_t = typename BinOp::type_t; using vector_t = vector; // data accessor needs to be this type @@ -25,22 +77,38 @@ struct scan NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; - // TODO: I think we need separate DataAccessor classes between device-wide and workgroup scan + NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; // TODO: double check op, it's all plus right now template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction - { - // TODO: what to do with config? - + { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); - // TODO: workgroup scan here, don't want to deal with that now + scalar_t currGroupReduction; + { + using data_proxy_t = WorkgroupDataProxy; + data_proxy_t wgDataAccessor = data_proxy_t::create(pc.pInputBuf, pc.pOutputBuf[Binop::BindingIndex]); + wgDataAccessor.preload(); + + if (Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + else + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done + + // TODO: double check this but it should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) + currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][config_t::ItemsPerInvocation_0-1u]; + } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - const scalar_t currGroupReduction = // TODO get last elem of curr workgroup; if (!invocIx) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; @@ -82,8 +150,14 @@ struct scan scratchAccessor.get(0u, prevReduction); prevReduction += currGroupReduction; - // TODO: propagate correctly when data accessor done - dataAccessor[invocIx] += prevReduction; + NBL_UNROLL + for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) + { + dtype_t data; + dataAccessor.template get(idx * WorkgroupSize + invocIx, data); + data += prevReduction; + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + } } }; } @@ -94,10 +168,10 @@ struct inclusive_scan using scalar_t = typename BinOp::type_t; template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) { - impl::scan fn; - fn.template __call(dataAccessor, scratchAccessor); + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); } }; @@ -107,8 +181,10 @@ struct exclusive_scan using scalar_t = typename BinOp::type_t; template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) { + impl::Scan fn; + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); } }; From 41928263948fb70097c534b09d78995fd3467802 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 15 Jul 2026 16:45:34 +0700 Subject: [PATCH 03/31] use correct binop --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 3903816c17..5fbf7aef07 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -69,6 +69,7 @@ struct Scan { using scalar_t = typename BinOp::type_t; using vector_t = vector; // data accessor needs to be this type + using binop_t = BinOp; NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_NotReady = 0; NBL_CONSTEXPR_STATIC_INLINE uint32_t Flag_Reduction = 1; // workgroup only has local reduction ready @@ -79,14 +80,13 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - - // TODO: double check op, it's all plus right now template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); + binop_t binop; scalar_t currGroupReduction; { @@ -105,7 +105,7 @@ struct Scan // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][config_t::ItemsPerInvocation_0-1u]; + currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -128,11 +128,10 @@ struct Scan if ((flagPayload & Flag_Mask) > Flag_NotReady) { - prevReduction += flagPayload >> Flag_Shift; + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); if ((flagPayload & Flag_Mask) == Flag_Inclusive) { - const scalar_t groupReduction = // TODO get last elem of workgroup index lookbackIx; - const scalar_t storeVal = Flag_Inclusive | ((prevReduction + groupReduction) << Flag_Shift); + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); scratchAccessor.set(0u, prevReduction); break; @@ -148,14 +147,16 @@ struct Scan scalar_t prevReduction; scratchAccessor.get(0u, prevReduction); - prevReduction += currGroupReduction; + prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) { dtype_t data; dataAccessor.template get(idx * WorkgroupSize + invocIx, data); - data += prevReduction; + NBL_UNROLL + for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) + data[i] = binop(prevReduction, data[i]); dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } From b132a2ee67dbd2d0858b3a1c4a675324ac7b3eec Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 16 Jul 2026 10:59:46 +0700 Subject: [PATCH 04/31] minor fix to creating wg accessor --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 5fbf7aef07..46c5081f7b 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -91,7 +91,7 @@ struct Scan scalar_t currGroupReduction; { using data_proxy_t = WorkgroupDataProxy; - data_proxy_t wgDataAccessor = data_proxy_t::create(pc.pInputBuf, pc.pOutputBuf[Binop::BindingIndex]); + data_proxy_t wgDataAccessor = data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); wgDataAccessor.preload(); if (Exclusive) From b46cad2296c029ff4f296281b24aa06111a759c4 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 16 Jul 2026 16:51:31 +0700 Subject: [PATCH 05/31] use a separate accessor type for reduction buf --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 46c5081f7b..2e7e24e740 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -81,8 +81,8 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - template - void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) // TODO: need different type for workgroupReduction + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); const uint16_t workgroupId = glsl::gl_WorkGroupID(); @@ -168,8 +168,9 @@ struct inclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) + // TODO: might want new concept for ReductionAccessor + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); @@ -181,8 +182,8 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(DataAccessor) workgroupReduction) + template && ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); From 59219c5df5e7d275d1037f4b736266c60f673e1a Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 17 Jul 2026 16:46:29 +0700 Subject: [PATCH 06/31] fix some compilation bugs, still not running correct --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 2e7e24e740..06523822ac 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -1,7 +1,7 @@ #ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ #define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ -#include "nbl/builtin/hlsl/workgroup2/shared_scan.hlsl" +#include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" namespace nbl { @@ -85,7 +85,7 @@ struct Scan void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - const uint16_t workgroupId = glsl::gl_WorkGroupID(); + const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; scalar_t currGroupReduction; @@ -133,7 +133,7 @@ struct Scan { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); break; } else @@ -146,30 +146,30 @@ struct Scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); scalar_t prevReduction; - scratchAccessor.get(0u, prevReduction); + scratchAccessor.template get(0u, prevReduction); prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) { - dtype_t data; - dataAccessor.template get(idx * WorkgroupSize + invocIx, data); + vector_t data; + dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); - dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } }; } -template) +template) struct inclusive_scan { using scalar_t = typename BinOp::type_t; // TODO: might want new concept for ReductionAccessor - template && ArithmeticSharedMemoryAccessor) + template && workgroup2::ArithmeticSharedMemoryAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; @@ -177,12 +177,12 @@ struct inclusive_scan } }; -template) +template) struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor) + template && workgroup2::ArithmeticSharedMemoryAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) { impl::Scan fn; From a056aeba3a5c4549d928186676e23a130071757b Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 21 Jul 2026 12:05:22 +0700 Subject: [PATCH 07/31] fix some bugs in chained scan, still need to add lookback fallback --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 06523822ac..a303825d03 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -88,28 +88,28 @@ struct Scan const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; + using wg_data_proxy_t = WorkgroupDataProxy; + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); scalar_t currGroupReduction; { - using data_proxy_t = WorkgroupDataProxy; - data_proxy_t wgDataAccessor = data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); wgDataAccessor.preload(); - if (Exclusive) - workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); + NBL_IF_CONSTEXPR(Exclusive) + workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); else - workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); + workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done + // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (!invocIx) + if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); @@ -121,7 +121,7 @@ struct Scan scalar_t prevReduction = 0u; uint16_t lookbackIx = workgroupId - uint16_t(1u); - while (true) // TODO: check if lookbackIx < 0? + while (lookbackIx > 0) // TODO: check if lookbackIx < 0? { scalar_t flagPayload; workgroupReduction.get(lookbackIx, flagPayload); @@ -139,8 +139,8 @@ struct Scan else lookbackIx--; } - else - lookbackIx--; + // else + // lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -150,9 +150,9 @@ struct Scan prevReduction = binop(prevReduction, currGroupReduction); NBL_UNROLL - for (uint16_t idx = 0; idx < ItemsPerInvoc; idx++) + for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) { - vector_t data; + vector_t data = wgDataAccessor.preloaded[idx]; dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) From 12b2d9bc5530194a8b6db8eb1241513fac4570e4 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 21 Jul 2026 16:51:34 +0700 Subject: [PATCH 08/31] lookback fallback to avoid spin --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index a303825d03..1b824ffcd9 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -3,6 +3,8 @@ #include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" +groupshared bool sIsLocked; + namespace nbl { namespace hlsl @@ -20,10 +22,10 @@ struct WorkgroupDataProxy NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; - static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf) + static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; - const uint32_t workgroupOffset = glsl::gl_WorkGroupID().x * VirtualWorkgroupSize * sizeof(dtype_t); + const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); return retval; } @@ -88,8 +90,11 @@ struct Scan const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); binop_t binop; + if (!invocIx) + sIsLocked = true; + using wg_data_proxy_t = WorkgroupDataProxy; - wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr()); + wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); scalar_t currGroupReduction; { wgDataAccessor.preload(); @@ -116,31 +121,80 @@ struct Scan } // lookback - if (workgroupId && !invocIx) + if (workgroupId > 0) { + bool locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + scalar_t prevReduction = 0u; uint16_t lookbackIx = workgroupId - uint16_t(1u); - while (lookbackIx > 0) // TODO: check if lookbackIx < 0? + while (locked) { - scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); + // lookback: try to get reduction from previous workgroups + if (!invocIx) + { + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); - if ((flagPayload & Flag_Mask) > Flag_NotReady) + if ((flagPayload & Flag_Mask) > Flag_NotReady) + { + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; + } + + // lookback failed + // broadcast id and prepare to do reduction ourselves + scratchAccessor.template set(1u, lookbackIx); + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + if (locked) { - prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); - if ((flagPayload & Flag_Mask) == Flag_Inclusive) + // do reduction for lookbackIx workgroup + uint32_t fallbackGroupId; + scratchAccessor.template get(1u, fallbackGroupId); + + wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); + fallbackDataAccessor.preload(); + scalar_t fallbackReduction = workgroup2::reduction::template __call(fallbackDataAccessor, scratchAccessor); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + if (!invocIx) { - const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); - break; + scalar_t fallbackPayload; + const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); + workgroupReduction.atomicMax(workgroupId, storeVal); + + prevReduction = binop(prevReduction, hlsl::max(fallbackReduction, fallbackPayload) >> Flag_Shift); + if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + } + else + { + lookbackIndex--; + } } - else - lookbackIx--; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + locked = sIsLocked; + scratchAccessor.workgroupExecutionAndMemoryBarrier(); } - // else - // lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); From 068776e951a198180a08b8041e64d292ff047399 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 10:54:47 +0700 Subject: [PATCH 09/31] implement atomic min and max --- .../nbl/builtin/hlsl/glsl_compat/core.hlsl | 104 ++++++++++++++++-- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index c2450e57aa..f2934fa762 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -116,16 +116,6 @@ enable_if_t, T> atomicXor(Ptr_T ptr, T value) { return spirv::atomicXor(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } -/* TODO: @Hazardu struct dispatchers like for `bitfieldExtract` -template -T atomicMin(NBL_REF_ARG(T) ptr, T value) -{ -} -template -T atomicMax(NBL_REF_ARG(T) ptr, T value) -{ -} -*/ template T atomicExchange(NBL_REF_ARG(T) ptr, T value) { @@ -147,6 +137,100 @@ enable_if_t, T> atomicCompSwap(Ptr_T ptr, T comparato return spirv::atomicCompareExchange(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, spv::MemorySemanticsMaskNone, value, comparator); } +namespace impl +{ +template +struct atomicMin; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMin) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMin(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +struct atomicMax; + +template +NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicSMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; + +template +NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral) +struct atomicMax) > +{ + static T __call(NBL_REF_ARG(T) ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } + + template // DXC Workaround + static T __call(Ptr_T ptr, T value) + { + return spirv::atomicUMax(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); + } +}; +} + +template +T atomicMin(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMin.__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMin(Ptr_T ptr, T value) +{ + return impl::atomicMin.template __call(ptr, value); +} +template +T atomicMax(NBL_REF_ARG(T) ptr, T value) +{ + return impl::atomicMax.__call(ptr, value); +} +template // DXC Workaround +enable_if_t, T> atomicMax(Ptr_T ptr, T value) +{ + return impl::atomicMax.template __call(ptr, value); +} + /** * GLSL extended math */ From 747d35b0630f85a6ff432345e8061683ebd570b0 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 11:27:38 +0700 Subject: [PATCH 10/31] some bug fixes to atomic min/max --- include/nbl/builtin/hlsl/glsl_compat/core.hlsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index f2934fa762..1549ce1990 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -213,22 +213,22 @@ struct atomicMax) > template T atomicMin(NBL_REF_ARG(T) ptr, T value) { - return impl::atomicMin.__call(ptr, value); + return impl::atomicMin::__call(ptr, value); } template // DXC Workaround enable_if_t, T> atomicMin(Ptr_T ptr, T value) { - return impl::atomicMin.template __call(ptr, value); + return impl::atomicMin::template __call(ptr, value); } template T atomicMax(NBL_REF_ARG(T) ptr, T value) { - return impl::atomicMax.__call(ptr, value); + return impl::atomicMax::__call(ptr, value); } template // DXC Workaround enable_if_t, T> atomicMax(Ptr_T ptr, T value) { - return impl::atomicMax.template __call(ptr, value); + return impl::atomicMax::template __call(ptr, value); } /** From 972a8fbddf9c3d9e0fb7cc7bfd0670d07623dcbe Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 14:41:10 +0700 Subject: [PATCH 11/31] a number of bug fixes to indexing stuff --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 1b824ffcd9..90e0bf9827 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -109,12 +109,17 @@ struct Scan // TODO: double check this but it should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly - if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) - currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (invocIx == lastInvocIx) + scratchAccessor.template set(0u, currGroupReduction); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + scratchAccessor.template get(0u, currGroupReduction); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (invocIx == glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u) + if (!invocIx) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); @@ -132,6 +137,7 @@ struct Scan while (locked) { // lookback: try to get reduction from previous workgroups + bool lookbackFailed = true; if (!invocIx) { scalar_t flagPayload; @@ -146,15 +152,15 @@ struct Scan workgroupReduction.atomicExchange(workgroupId, storeVal); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; - break; + lookbackFailed = false; } else lookbackIx--; } - // lookback failed // broadcast id and prepare to do reduction ourselves - scratchAccessor.template set(1u, lookbackIx); + if (lookbackFailed) + scratchAccessor.template set(1u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -163,7 +169,7 @@ struct Scan if (locked) { // do reduction for lookbackIx workgroup - uint32_t fallbackGroupId; + uint16_t fallbackGroupId; scratchAccessor.template get(1u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); @@ -173,11 +179,10 @@ struct Scan if (!invocIx) { - scalar_t fallbackPayload; const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); - workgroupReduction.atomicMax(workgroupId, storeVal); + const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); - prevReduction = binop(prevReduction, hlsl::max(fallbackReduction, fallbackPayload) >> Flag_Shift); + prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); @@ -187,7 +192,7 @@ struct Scan } else { - lookbackIndex--; + lookbackIx--; } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -199,15 +204,14 @@ struct Scan } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - scalar_t prevReduction; - scratchAccessor.template get(0u, prevReduction); - prevReduction = binop(prevReduction, currGroupReduction); + scalar_t prevReduction = 0u; + if (workgroupId > 0) + scratchAccessor.template get(0u, prevReduction); NBL_UNROLL for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) { vector_t data = wgDataAccessor.preloaded[idx]; - dataAccessor.template get(idx * WorkgroupSize + invocIx, data); NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); From 9085b5dd60cf98d2c70ff73cbef4d5b13b001f1e Mon Sep 17 00:00:00 2001 From: keptsecret Date: Wed, 22 Jul 2026 16:59:54 +0700 Subject: [PATCH 12/31] fix exclusive scan to reduction store --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 90e0bf9827..4a3f05e7a1 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -99,7 +99,14 @@ struct Scan { wgDataAccessor.preload(); - NBL_IF_CONSTEXPR(Exclusive) + // TODO: double check this but it should be the last element of the last workgroup thread + // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly + const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + scalar_t lastElem; + if (invocIx == lastInvocIx) + lastElem = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + + if (Exclusive) workgroup2::exclusive_scan::template __call(wgDataAccessor, scratchAccessor); else workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); @@ -107,10 +114,9 @@ struct Scan // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done - // TODO: double check this but it should be the last element of the last workgroup thread - // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly - const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (Exclusive) + currGroupReduction = binop(currGroupReduction, lastElem); if (invocIx == lastInvocIx) scratchAccessor.template set(0u, currGroupReduction); scratchAccessor.workgroupExecutionAndMemoryBarrier(); From 3a59e93ae2be7bd99d44f2b27ed6b17957fd4fa7 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 23 Jul 2026 17:03:45 +0700 Subject: [PATCH 13/31] use a workgroup counter to determine execution order --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 4a3f05e7a1..8e8e005bae 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -83,16 +83,23 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; - template - void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template + void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - const uint16_t workgroupId = uint16_t(glsl::gl_WorkGroupID().x); - binop_t binop; - if (!invocIx) + { + const uint32_t id = workgroupCounter.atomicAdd(0u, 1u); + scratchAccessor.template set(0u, id); sIsLocked = true; + } + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + uint16_t workgroupId; + scratchAccessor.template get(0u, workgroupId); + scratchAccessor.workgroupExecutionAndMemoryBarrier(); + + binop_t binop; using wg_data_proxy_t = WorkgroupDataProxy; wg_data_proxy_t wgDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), workgroupId); scalar_t currGroupReduction; @@ -210,6 +217,8 @@ struct Scan } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + dataAccessor.initAtWorkgroupID(workgroupId); + scalar_t prevReduction = 0u; if (workgroupId > 0) scratchAccessor.template get(0u, prevReduction); @@ -233,11 +242,11 @@ struct inclusive_scan using scalar_t = typename BinOp::type_t; // TODO: might want new concept for ReductionAccessor - template && workgroup2::ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template && workgroup2::ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; - fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); } }; @@ -246,11 +255,11 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && workgroup2::ArithmeticSharedMemoryAccessor) - static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction) + template && workgroup2::ArithmeticSharedMemoryAccessor) + static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; - fn.template __call(dataAccessor, scratchAccessor, workgroupReduction); + fn.template __call(dataAccessor, scratchAccessor, workgroupReduction, counter); } }; From 25e2d0193c0a6a61c5e9c2d29daa806be1e5fc7f Mon Sep 17 00:00:00 2001 From: keptsecret Date: Mon, 27 Jul 2026 12:23:45 +0700 Subject: [PATCH 14/31] fix for indexing in workgroup scan accessor (unsure if temp fix) --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 8e8e005bae..2c064f0084 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -19,14 +19,15 @@ struct WorkgroupDataProxy { using dtype_t = vector; - NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << WorkgroupSizeLog2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = VirtualWorkgroupSize / WorkgroupSize; + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t PreloadedDataCount = uint16_t(VirtualWorkgroupSize / WorkgroupSize); static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); - retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf/* + workgroupOffset*/, outputBuf /*+ workgroupOffset*/); + retval.workgroupID = workgroupId; return retval; } @@ -46,14 +47,14 @@ struct WorkgroupDataProxy const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.get(idx * WorkgroupSize + invocIx, preloaded[idx]); + accessor.get((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); } void unload() { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.set(idx * WorkgroupSize + invocIx, preloaded[idx]); + accessor.set((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); } void workgroupExecutionAndMemoryBarrier() @@ -64,6 +65,7 @@ struct WorkgroupDataProxy DoubleLegacyBdaAccessor accessor; dtype_t preloaded[PreloadedDataCount]; + uint32_t workgroupID; // TODO: remove possibly?, current problem is that adding the workgroup offset directly to address doesn't work }; template // TODO: Config is same as workgroup2 stuff? @@ -80,8 +82,8 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t WorkgroupSize = uint16_t(1u) << Config::WorkgroupSizeLog2; - NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = Config::VirtualWorkgroupSize / WorkgroupSize; + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; + NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) @@ -94,6 +96,7 @@ struct Scan sIsLocked = true; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + // spirv::memoryBarrier(spv::ScopeDevice, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask); uint16_t workgroupId; scratchAccessor.template get(0u, workgroupId); @@ -138,7 +141,6 @@ struct Scan workgroupReduction.atomicExchange(workgroupId, storeVal); } - // lookback if (workgroupId > 0) { bool locked = sIsLocked; @@ -230,7 +232,7 @@ struct Scan NBL_UNROLL for (uint16_t i = 0; i < Config::ItemsPerInvocation_0; i++) data[i] = binop(prevReduction, data[i]); - dataAccessor.template set(idx * WorkgroupSize + invocIx, data); + dataAccessor.template set(idx * WorkgroupSize + invocIx, data); } } }; From ba9e151427a8b92d96294bf987fa0314d76f2d83 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Tue, 28 Jul 2026 16:58:42 +0700 Subject: [PATCH 15/31] fix lookback to spin a bit for some more consistency --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 2c064f0084..d551005448 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -84,6 +84,7 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); + NBL_CONSTEXPR_STATIC_INLINE uint16_t MaxSpinCount = uint16_t(4u); template void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) @@ -152,29 +153,36 @@ struct Scan while (locked) { // lookback: try to get reduction from previous workgroups - bool lookbackFailed = true; if (!invocIx) { - scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); - - if ((flagPayload & Flag_Mask) > Flag_NotReady) + uint16_t spinCount = uint16_t(0u); + [loop] + while (spinCount < MaxSpinCount) { - prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); - if ((flagPayload & Flag_Mask) == Flag_Inclusive) + scalar_t flagPayload; + workgroupReduction.get(lookbackIx, flagPayload); + + if ((flagPayload & Flag_Mask) > Flag_NotReady) { - const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); - sIsLocked = false; - lookbackFailed = false; + spinCount = uint16_t(0u); + prevReduction = binop(prevReduction, flagPayload >> Flag_Shift); + if ((flagPayload & Flag_Mask) == Flag_Inclusive) + { + const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); + workgroupReduction.atomicExchange(workgroupId, storeVal); + scratchAccessor.template set(0u, prevReduction); + sIsLocked = false; + break; + } + else + lookbackIx--; } else - lookbackIx--; + spinCount++; } // broadcast id and prepare to do reduction ourselves - if (lookbackFailed) + if (spinCount == MaxSpinCount) scratchAccessor.template set(1u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -206,9 +214,7 @@ struct Scan sIsLocked = false; } else - { lookbackIx--; - } } scratchAccessor.workgroupExecutionAndMemoryBarrier(); From dd61813dc3c0890be850767324b9289b84a6a116 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 15:17:10 +0700 Subject: [PATCH 16/31] can use single scratch address + fixes bug in example, memorybarrier probably not needed but left in for now --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index d551005448..b20add8b7b 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -97,7 +97,6 @@ struct Scan sIsLocked = true; } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // spirv::memoryBarrier(spv::ScopeDevice, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask); uint16_t workgroupId; scratchAccessor.template get(0u, workgroupId); @@ -141,8 +140,9 @@ struct Scan const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); } + // workgroupReduction.memoryBarrier(); - if (workgroupId > 0) + if (workgroupId) { bool locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -183,9 +183,10 @@ struct Scan // broadcast id and prepare to do reduction ourselves if (spinCount == MaxSpinCount) - scratchAccessor.template set(1u, lookbackIx); + scratchAccessor.template set(0u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); + // workgroupReduction.memoryBarrier(); locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -193,7 +194,7 @@ struct Scan { // do reduction for lookbackIx workgroup uint16_t fallbackGroupId; - scratchAccessor.template get(1u, fallbackGroupId); + scratchAccessor.template get(0u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); fallbackDataAccessor.preload(); @@ -204,12 +205,14 @@ struct Scan { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); + // workgroupReduction.memoryBarrier(); prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); - if (!fallbackGroupId || (fallbackPayload & Flag_Mask) == Flag_Inclusive) + if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); + // workgroupReduction.memoryBarrier(); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; } @@ -228,7 +231,7 @@ struct Scan dataAccessor.initAtWorkgroupID(workgroupId); scalar_t prevReduction = 0u; - if (workgroupId > 0) + if (workgroupId) scratchAccessor.template get(0u, prevReduction); NBL_UNROLL From 02d6966fc7b42773b928c7844c616b36e9f2e929 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:40:33 +0700 Subject: [PATCH 17/31] fixes to indexing in preload --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index b20add8b7b..59b8b86726 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -25,8 +25,7 @@ struct WorkgroupDataProxy static WorkgroupDataProxy create(const uint64_t inputBuf, const uint64_t outputBuf, const uint16_t workgroupId) { WorkgroupDataProxy retval; - const uint32_t workgroupOffset = workgroupId * VirtualWorkgroupSize * sizeof(dtype_t); - retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf/* + workgroupOffset*/, outputBuf /*+ workgroupOffset*/); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf, outputBuf); retval.workgroupID = workgroupId; return retval; } @@ -47,14 +46,7 @@ struct WorkgroupDataProxy const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); NBL_UNROLL for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.get((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); - } - void unload() - { - const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - NBL_UNROLL - for (uint16_t idx = 0; idx < PreloadedDataCount; idx++) - accessor.set((workgroupID + idx) * WorkgroupSize + invocIx, preloaded[idx]); + accessor.get(workgroupID * VirtualWorkgroupSize + idx * WorkgroupSize + invocIx, preloaded[idx]); } void workgroupExecutionAndMemoryBarrier() @@ -109,7 +101,7 @@ struct Scan { wgDataAccessor.preload(); - // TODO: double check this but it should be the last element of the last workgroup thread + // should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; scalar_t lastElem; @@ -122,8 +114,6 @@ struct Scan workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // wgDataAccessor.unload(); // TODO: maybe we don't have to unload, just write once after everything is done - currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; if (Exclusive) currGroupReduction = binop(currGroupReduction, lastElem); From e5153a9aa09c6e91ebf26f39bae0a8b9e92d994d Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:53:52 +0700 Subject: [PATCH 18/31] add chain scan hlsl to cmakelist --- src/nbl/builtin/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index df20f6b6b3..a75768c981 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -362,6 +362,8 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/items_per_inv LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/impl/arithmetic_config_def.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/arithmetic.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/workgroup2/shared_scan.hlsl") +#device scan +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/scan/chained_scan.hlsl") #Extensions LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/SVertexAttributes.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/ext/FullScreenTriangle/default.vert.hlsl") From fbf4b57f82b044feee1729d117c260eca4372bb1 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Thu, 30 Jul 2026 16:54:02 +0700 Subject: [PATCH 19/31] latest example --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 8515f32fc4..0990b176a1 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 8515f32fc4bb630270dbe3bca83a2835fd2006e2 +Subproject commit 0990b176a17aee6381a767162f7c2d1d31335c8b From c5b30319c1091c5ec6d04ad545afd64245e750d5 Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 31 Jul 2026 12:15:16 +0700 Subject: [PATCH 20/31] concepts for accessors of device scan, removed obsolete comments --- .../concepts/accessors/device_arithmetic.hlsl | 65 +++++++++++++++++++ .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 28 ++++---- src/nbl/builtin/CMakeLists.txt | 1 + 3 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl new file mode 100644 index 0000000000..e852f889e6 --- /dev/null +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -0,0 +1,65 @@ +#ifndef _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ +#define _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_ + +#include "nbl/builtin/hlsl/concepts/accessors/generic_shared_data.hlsl" + +namespace nbl +{ +namespace hlsl +{ +namespace scan +{ + +template +NBL_BOOL_CONCEPT ArithmeticSharedMemoryAccessor = concepts::accessors::GenericSharedMemoryAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticReadOnlyDataAccessor = concepts::accessors::GenericReadAccessor; + +template +NBL_BOOL_CONCEPT ArithmeticDataAccessor = concepts::accessors::GenericDataAccessor; + +#define NBL_CONCEPT_NAME DeviceReductionsAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicExchange(index, val)), is_same_v, V)) +); +#undef val +#undef index +#undef accessor +#include + +// TODO: as counter, maybe just increment 1 always? +#define NBL_CONCEPT_NAME WorkgroupCounterAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, uint32_t) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicAdd(index, val)), is_same_v, uint32_t)) +); +#undef val +#undef index +#undef accessor +#include + +} +} +} + +#endif diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 59b8b86726..2f1c10cd95 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -1,6 +1,7 @@ #ifndef _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ #define _NBL_HLSL_SCAN_CHAINED_SCAN_INCLUDED_ +#include "nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl" #include "nbl/builtin/hlsl/workgroup2/arithmetic.hlsl" groupshared bool sIsLocked; @@ -57,7 +58,7 @@ struct WorkgroupDataProxy DoubleLegacyBdaAccessor accessor; dtype_t preloaded[PreloadedDataCount]; - uint32_t workgroupID; // TODO: remove possibly?, current problem is that adding the workgroup offset directly to address doesn't work + uint32_t workgroupID; }; template // TODO: Config is same as workgroup2 stuff? @@ -118,10 +119,10 @@ struct Scan if (Exclusive) currGroupReduction = binop(currGroupReduction, lastElem); if (invocIx == lastInvocIx) - scratchAccessor.template set(0u, currGroupReduction); + scratchAccessor.template set(0u, currGroupReduction); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - scratchAccessor.template get(0u, currGroupReduction); + scratchAccessor.template get(0u, currGroupReduction); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -130,7 +131,6 @@ struct Scan const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); } - // workgroupReduction.memoryBarrier(); if (workgroupId) { @@ -150,7 +150,7 @@ struct Scan while (spinCount < MaxSpinCount) { scalar_t flagPayload; - workgroupReduction.get(lookbackIx, flagPayload); + workgroupReduction.template get(lookbackIx, flagPayload); if ((flagPayload & Flag_Mask) > Flag_NotReady) { @@ -160,7 +160,7 @@ struct Scan { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - scratchAccessor.template set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); sIsLocked = false; break; } @@ -173,10 +173,9 @@ struct Scan // broadcast id and prepare to do reduction ourselves if (spinCount == MaxSpinCount) - scratchAccessor.template set(0u, lookbackIx); + scratchAccessor.template set(0u, lookbackIx); } scratchAccessor.workgroupExecutionAndMemoryBarrier(); - // workgroupReduction.memoryBarrier(); locked = sIsLocked; scratchAccessor.workgroupExecutionAndMemoryBarrier(); @@ -184,7 +183,7 @@ struct Scan { // do reduction for lookbackIx workgroup uint16_t fallbackGroupId; - scratchAccessor.template get(0u, fallbackGroupId); + scratchAccessor.template get(0u, fallbackGroupId); wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); fallbackDataAccessor.preload(); @@ -195,15 +194,13 @@ struct Scan { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); - // workgroupReduction.memoryBarrier(); prevReduction = binop(prevReduction, hlsl::mix(fallbackReduction, fallbackPayload >> Flag_Shift, fallbackPayload > scalar_t(0.0))); if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); workgroupReduction.atomicExchange(workgroupId, storeVal); - // workgroupReduction.memoryBarrier(); - scratchAccessor.template set(0u, prevReduction); + scratchAccessor.template set(0u, prevReduction); sIsLocked = false; } else @@ -222,7 +219,7 @@ struct Scan scalar_t prevReduction = 0u; if (workgroupId) - scratchAccessor.template get(0u, prevReduction); + scratchAccessor.template get(0u, prevReduction); NBL_UNROLL for (uint16_t idx = 0; idx < wg_data_proxy_t::PreloadedDataCount; idx++) @@ -242,8 +239,7 @@ struct inclusive_scan { using scalar_t = typename BinOp::type_t; - // TODO: might want new concept for ReductionAccessor - template && workgroup2::ArithmeticSharedMemoryAccessor) + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; @@ -256,7 +252,7 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && workgroup2::ArithmeticSharedMemoryAccessor) + template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; diff --git a/src/nbl/builtin/CMakeLists.txt b/src/nbl/builtin/CMakeLists.txt index a75768c981..d6140e2625 100644 --- a/src/nbl/builtin/CMakeLists.txt +++ b/src/nbl/builtin/CMakeLists.txt @@ -387,6 +387,7 @@ LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/storable_i LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/generic_shared_data.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/fft.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/workgroup_arithmetic.hlsl") +LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/concepts/accessors/device_arithmetic.hlsl") #tgmath LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath.hlsl") LIST_BUILTIN_RESOURCE(NBL_RESOURCES_TO_EMBED "hlsl/tgmath/impl.hlsl") From 12a8f560d2a65bbc507829a952c36f6889cdba7e Mon Sep 17 00:00:00 2001 From: keptsecret Date: Fri, 31 Jul 2026 12:16:12 +0700 Subject: [PATCH 21/31] latest example --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 0990b176a1..5ac0f19fd4 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 0990b176a17aee6381a767162f7c2d1d31335c8b +Subproject commit 5ac0f19fd46cef5680b92b5fa4fd97f3f2b29fe4 From dc7bfa65fa221cc69497e4080a8a3a02511f6038 Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Thu, 8 Oct 2026 16:37:11 +0700 Subject: [PATCH 22/31] removed unnecessary workgroup barriers, added some clarifying comments and usage --- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 2f1c10cd95..22c6bcec58 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -61,7 +61,7 @@ struct WorkgroupDataProxy uint32_t workgroupID; }; -template // TODO: Config is same as workgroup2 stuff? +template struct Scan { using scalar_t = typename BinOp::type_t; @@ -75,6 +75,9 @@ struct Scan NBL_CONSTEXPR_STATIC_INLINE uint16_t Flag_Shift = 2; + // IMPORTANT NOTE: for binary operations scalar precision, we reserve two bits for packing extra flags + // thus, the user will only get 30-bit precision if they use uint32_t + NBL_CONSTEXPR_STATIC_INLINE uint32_t WorkgroupSize = 1u << Config::WorkgroupSizeLog2; NBL_CONSTEXPR_STATIC_INLINE uint16_t ItemsPerInvoc = uint16_t(Config::VirtualWorkgroupSize / WorkgroupSize); NBL_CONSTEXPR_STATIC_INLINE uint16_t MaxSpinCount = uint16_t(4u); @@ -83,7 +86,7 @@ struct Scan void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) workgroupCounter) { const uint16_t invocIx = workgroup::SubgroupContiguousIndex(); - if (!invocIx) + if (invocIx == 0u) { const uint32_t id = workgroupCounter.atomicAdd(0u, 1u); scratchAccessor.template set(0u, id); @@ -93,7 +96,6 @@ struct Scan uint16_t workgroupId; scratchAccessor.template get(0u, workgroupId); - scratchAccessor.workgroupExecutionAndMemoryBarrier(); binop_t binop; using wg_data_proxy_t = WorkgroupDataProxy; @@ -115,27 +117,27 @@ struct Scan workgroup2::inclusive_scan::template __call(wgDataAccessor, scratchAccessor); scratchAccessor.workgroupExecutionAndMemoryBarrier(); - currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; - if (Exclusive) - currGroupReduction = binop(currGroupReduction, lastElem); if (invocIx == lastInvocIx) + { + currGroupReduction = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; + if (Exclusive) + currGroupReduction = binop(currGroupReduction, lastElem); scratchAccessor.template set(0u, currGroupReduction); + } scratchAccessor.workgroupExecutionAndMemoryBarrier(); scratchAccessor.template get(0u, currGroupReduction); } - scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (!invocIx) + if (invocIx == 0u) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; workgroupReduction.atomicExchange(workgroupId, storeVal); } - if (workgroupId) + if (workgroupId > 0u) { bool locked = sIsLocked; - scratchAccessor.workgroupExecutionAndMemoryBarrier(); scalar_t prevReduction = 0u; uint16_t lookbackIx = workgroupId - uint16_t(1u); @@ -143,7 +145,7 @@ struct Scan while (locked) { // lookback: try to get reduction from previous workgroups - if (!invocIx) + if (invocIx == 0u) { uint16_t spinCount = uint16_t(0u); [loop] @@ -178,7 +180,8 @@ struct Scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); locked = sIsLocked; - scratchAccessor.workgroupExecutionAndMemoryBarrier(); + // Fallback path: we spun to MaxSpinCount, but no previous workgroup had their global reduction ready (Flag_Inclusive). + // So we try to do reduction for all previous work groups one by one until we find a Flag_Inclusive if (locked) { // do reduction for lookbackIx workgroup @@ -188,9 +191,8 @@ struct Scan wg_data_proxy_t fallbackDataAccessor = wg_data_proxy_t::create(dataAccessor.getInputBufAddr(), dataAccessor.getOutputBufAddr(), fallbackGroupId); fallbackDataAccessor.preload(); scalar_t fallbackReduction = workgroup2::reduction::template __call(fallbackDataAccessor, scratchAccessor); - scratchAccessor.workgroupExecutionAndMemoryBarrier(); - if (!invocIx) + if (invocIx == 0u) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, fallbackGroupId > 0u) | (fallbackReduction << Flag_Shift); const scalar_t fallbackPayload = workgroupReduction.atomicMax(fallbackGroupId, storeVal); @@ -209,7 +211,6 @@ struct Scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); locked = sIsLocked; - scratchAccessor.workgroupExecutionAndMemoryBarrier(); } } } From a94a8ca355de914b92fde0ad345918fa140add94 Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Fri, 9 Oct 2026 14:23:49 +0700 Subject: [PATCH 23/31] add spirv atomic load/store ops --- .../nbl/builtin/hlsl/spirv_intrinsics/core.hlsl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl b/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl index 3497babfeb..e0449fa5e5 100644 --- a/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl +++ b/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl @@ -255,6 +255,22 @@ template // DXC Workaround [[vk::ext_instruction(spv::OpAtomicCompareExchange)]] enable_if_t, T> atomicCompareExchange(Ptr_T ptr, uint32_t memoryScope, uint32_t memSemanticsEqual, uint32_t memSemanticsUnequal, T value, T comparator); +template +[[vk::ext_instruction(spv::OpAtomicLoad)]] +T atomicLoad([[vk::ext_reference]] T ptr, uint32_t memoryScope, uint32_t memorySemantics); + +template // DXC Workaround +[[vk::ext_instruction(spv::OpAtomicLoad)]] +enable_if_t, T> atomicLoad(Ptr_T ptr, uint32_t memoryScope, uint32_t memorySemantics); + +template +[[vk::ext_instruction(spv::OpAtomicStore)]] +enable_if_t, void> atomicStore([[vk::ext_reference]] T ptr, uint32_t memoryScope, uint32_t memorySemantics, T value); + +template // DXC Workaround +[[vk::ext_instruction(spv::OpAtomicStore)]] +enable_if_t && is_scalar_v, void> atomicStore(Ptr_T ptr, uint32_t memoryScope, uint32_t memorySemantics, T value); + template __NBL_CAPABILITY_PhysicalStorageBufferAddresses From ad803f37d4350ddfef788919393e5136c420366b Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Fri, 9 Oct 2026 14:24:47 +0700 Subject: [PATCH 24/31] use atomic store instead of exchange --- .../builtin/hlsl/concepts/accessors/device_arithmetic.hlsl | 3 +-- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl index e852f889e6..9d991202cc 100644 --- a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -32,14 +32,13 @@ NBL_CONCEPT_BEGIN(3) NBL_CONCEPT_END( ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) - ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicExchange(index, val)), is_same_v, V)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicStore(index, val)), is_same_v, void)) ); #undef val #undef index #undef accessor #include -// TODO: as counter, maybe just increment 1 always? #define NBL_CONCEPT_NAME WorkgroupCounterAccessor #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) #define NBL_CONCEPT_TPLT_PRM_NAMES (T) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 22c6bcec58..ef254091c1 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -132,7 +132,7 @@ struct Scan if (invocIx == 0u) { const scalar_t storeVal = hlsl::mix(Flag_Inclusive, Flag_Reduction, workgroupId > 0u) | currGroupReduction << Flag_Shift; - workgroupReduction.atomicExchange(workgroupId, storeVal); + workgroupReduction.atomicStore(workgroupId, storeVal); } if (workgroupId > 0u) @@ -161,7 +161,7 @@ struct Scan if ((flagPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); + workgroupReduction.atomicStore(workgroupId, storeVal); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; break; @@ -201,7 +201,7 @@ struct Scan if (fallbackGroupId == 0u || (fallbackPayload & Flag_Mask) == Flag_Inclusive) { const scalar_t storeVal = Flag_Inclusive | (binop(prevReduction, currGroupReduction) << Flag_Shift); - workgroupReduction.atomicExchange(workgroupId, storeVal); + workgroupReduction.atomicStore(workgroupId, storeVal); scratchAccessor.template set(0u, prevReduction); sIsLocked = false; } From ba1466e2adeccf50f90de3ab029c9a3cd38dd6f4 Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Fri, 9 Oct 2026 15:39:37 +0700 Subject: [PATCH 25/31] generalize the concepts for atomic accessors --- .../concepts/accessors/device_arithmetic.hlsl | 31 ++++++++++++------- .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 4 +-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl index 9d991202cc..c1989e7dd0 100644 --- a/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl @@ -10,14 +10,24 @@ namespace hlsl namespace scan { -template -NBL_BOOL_CONCEPT ArithmeticSharedMemoryAccessor = concepts::accessors::GenericSharedMemoryAccessor; - -template -NBL_BOOL_CONCEPT ArithmeticReadOnlyDataAccessor = concepts::accessors::GenericReadAccessor; - -template -NBL_BOOL_CONCEPT ArithmeticDataAccessor = concepts::accessors::GenericDataAccessor; +#define NBL_CONCEPT_NAME GenericAtomicAccessor +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V) +#define NBL_CONCEPT_PARAM_0 (accessor, T) +#define NBL_CONCEPT_PARAM_1 (val, V) +#define NBL_CONCEPT_PARAM_2 (index, uint64_t) +NBL_CONCEPT_BEGIN(3) +#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicStore(index, val)), is_same_v, void)) +); +#undef val +#undef index +#undef accessor +#include #define NBL_CONCEPT_NAME DeviceReductionsAccessor #define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) @@ -30,16 +40,15 @@ NBL_CONCEPT_BEGIN(3) #define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 #define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 NBL_CONCEPT_END( - ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t)) + ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(GenericAtomicAccessor, T, V)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V)) - ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicStore(index, val)), is_same_v, void)) ); #undef val #undef index #undef accessor #include -#define NBL_CONCEPT_NAME WorkgroupCounterAccessor +#define NBL_CONCEPT_NAME GenericAtomicArithmeticAccessor #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) #define NBL_CONCEPT_TPLT_PRM_NAMES (T) #define NBL_CONCEPT_PARAM_0 (accessor, T) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index ef254091c1..0a58346b7b 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -240,7 +240,7 @@ struct inclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) + template && concepts::accessors::GenericSharedMemoryAccessor && DeviceReductionsAccessor && GenericAtomicArithmeticAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; @@ -253,7 +253,7 @@ struct exclusive_scan { using scalar_t = typename BinOp::type_t; - template && ArithmeticSharedMemoryAccessor && DeviceReductionsAccessor && WorkgroupCounterAccessor) + template && concepts::accessors::GenericSharedMemoryAccessor && DeviceReductionsAccessor && GenericAtomicArithmeticAccessor) static void __call(NBL_REF_ARG(DataAccessor) dataAccessor, NBL_REF_ARG(ScratchAccessor) scratchAccessor, NBL_REF_ARG(ReductionAccessor) workgroupReduction, NBL_REF_ARG(WorkgroupCounter) counter) { impl::Scan fn; From b07782f75268ad36ebd1626fbefb9fdc375d6abd Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Fri, 9 Oct 2026 16:23:19 +0700 Subject: [PATCH 26/31] add comment about ongoing issue with dxc for atomics ptr scope --- include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl b/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl index e0449fa5e5..4ad4b442d6 100644 --- a/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl +++ b/include/nbl/builtin/hlsl/spirv_intrinsics/core.hlsl @@ -145,6 +145,9 @@ template [[vk::ext_instruction(spv::OpCopyLogical)]] enable_if_t/* && !is_same_v*/,T> copyLogical(Ptr_U v); +// Atomics still don't detect the address space of `ptr` and narrow down the sync-scope properly, hence the DXC workarounds +// see issue: https://github.com/microsoft/DirectXShaderCompiler/issues/6508 -- same as in glsl_compat/core.hlsl + // Here's the thing with atomics, it's not only the data type that dictates whether you can do an atomic or not. // It's the storage class that has the most effect (shared vs storage vs image) and we can't check that easily template // integers operate on 2s complement so same op for signed and unsigned From 4cb1b2acddbe530d067318cc4e2fbe8e8148b385 Mon Sep 17 00:00:00 2001 From: Sorakrit Chonwattanagul Date: Mon, 12 Oct 2026 14:31:10 +0700 Subject: [PATCH 27/31] use binop identity to initialize prevReduction --- include/nbl/builtin/hlsl/scan/chained_scan.hlsl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index 0a58346b7b..ad2a2335ca 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -139,7 +139,7 @@ struct Scan { bool locked = sIsLocked; - scalar_t prevReduction = 0u; + scalar_t prevReduction = binop_t::identity; uint16_t lookbackIx = workgroupId - uint16_t(1u); while (locked) @@ -218,7 +218,7 @@ struct Scan dataAccessor.initAtWorkgroupID(workgroupId); - scalar_t prevReduction = 0u; + scalar_t prevReduction = binop_t::identity; if (workgroupId) scratchAccessor.template get(0u, prevReduction); From b9d550f0488dcb22d9c592cf98fc05e7f3a10b8d Mon Sep 17 00:00:00 2001 From: Erfan Ahmadi Date: Thu, 22 Oct 2026 17:25:41 +0400 Subject: [PATCH 28/31] use spirv intrinsics directly for builtins and clean up glsl_compat usage Document and add TODOs for moving non-GLSL utilities out of the glsl namespace and renaming it to glsl_compat. Emitted SPIR-V is unchanged. --- examples_tests | 2 +- .../nbl/builtin/hlsl/bda/bda_accessor.hlsl | 6 +-- .../builtin/hlsl/bda/legacy_bda_accessor.hlsl | 2 +- .../builtin/hlsl/blit/default_blit.comp.hlsl | 2 +- .../builtin/hlsl/emulated/float64_t_impl.hlsl | 1 - include/nbl/builtin/hlsl/functional.hlsl | 2 +- .../nbl/builtin/hlsl/glsl_compat/core.hlsl | 14 +++++- .../hlsl/glsl_compat/subgroup_arithmetic.hlsl | 2 + .../hlsl/glsl_compat/subgroup_basic.hlsl | 1 + include/nbl/builtin/hlsl/memory_accessor.hlsl | 1 - .../builtin/hlsl/prefix_sum_blur/blur.hlsl | 1 - .../nbl/builtin/hlsl/scan/chained_scan.hlsl | 4 +- include/nbl/builtin/hlsl/sort/counting.hlsl | 2 +- .../builtin/hlsl/spirv_intrinsics/core.hlsl | 2 + .../subgroup/arithmetic_portability_impl.hlsl | 7 +-- include/nbl/builtin/hlsl/subgroup/ballot.hlsl | 5 +- include/nbl/builtin/hlsl/subgroup/basic.hlsl | 2 - include/nbl/builtin/hlsl/subgroup/fft.hlsl | 19 ++++---- .../arithmetic_portability_impl.hlsl | 5 +- .../nbl/builtin/hlsl/subgroup2/ballot.hlsl | 5 +- include/nbl/builtin/hlsl/surface_transform.h | 1 - .../builtin/hlsl/workgroup/arithmetic.hlsl | 2 +- .../nbl/builtin/hlsl/workgroup/ballot.hlsl | 8 ++-- include/nbl/builtin/hlsl/workgroup/basic.hlsl | 5 +- include/nbl/builtin/hlsl/workgroup/fft.hlsl | 20 ++++---- .../builtin/hlsl/workgroup/shared_scan.hlsl | 13 +++--- .../hlsl/workgroup2/arithmetic_config.hlsl | 2 +- .../builtin/hlsl/workgroup2/shared_scan.hlsl | 46 ++++++++++--------- 28 files changed, 102 insertions(+), 80 deletions(-) diff --git a/examples_tests b/examples_tests index d19303f0ea..105f94003a 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit d19303f0eab48369e728e2058ca0473cad05d392 +Subproject commit 105f94003a2b11c7ab3b349d6a8cb6474ca5cbff diff --git a/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl b/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl index caefc814c5..00ad9f16d8 100644 --- a/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl +++ b/include/nbl/builtin/hlsl/bda/bda_accessor.hlsl @@ -4,7 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/bda/__ptr.hlsl" namespace nbl @@ -66,7 +66,7 @@ struct BdaAccessor : impl::BdaAccessorBase atomicAdd(const uint64_t index, const T value) { bda::__ptr target = ptr + index; - return glsl::atomicAdd(target.template deref().ptr.value, value); + return spirv::atomicIAdd(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } template @@ -74,7 +74,7 @@ struct BdaAccessor : impl::BdaAccessorBase atomicSub(const uint64_t index, const T value) { bda::__ptr target = ptr + index; - return glsl::atomicSub(target.template deref().ptr.value, value); + return spirv::atomicISub(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value); } bda::__ptr ptr; diff --git a/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl b/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl index f778e2b364..a3fb51ef94 100644 --- a/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl +++ b/include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl @@ -9,7 +9,7 @@ #ifndef _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/bda/__ptr.hlsl" diff --git a/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl b/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl index 4b97bbc08f..5eb2457612 100644 --- a/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl +++ b/include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl @@ -67,7 +67,7 @@ void main() OutImgAccessor outImgA; outImgA.descIx = pc.outputDescIx; - const uint16_t3 wgID = _static_cast(glsl::gl_WorkGroupID()); + const uint16_t3 wgID = _static_cast(spirv::WorkgroupId); const uint16_t3 baseCoord = pc.perWG.getOutputBaseCoord(wgID); // TODO: If and when someone can be bothered, change the blit api to compile a pipeline per image dimension, maybe it will be faster switch (pc.perWG.imageDim) diff --git a/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl b/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl index df785e3e8f..bcba437131 100644 --- a/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl +++ b/include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl @@ -4,7 +4,6 @@ #include #include #include -#include #include // TODO: when it will be possible, use this unions wherever they fit: diff --git a/include/nbl/builtin/hlsl/functional.hlsl b/include/nbl/builtin/hlsl/functional.hlsl index 118fe07c63..8c80b6b273 100644 --- a/include/nbl/builtin/hlsl/functional.hlsl +++ b/include/nbl/builtin/hlsl/functional.hlsl @@ -5,7 +5,7 @@ #define _NBL_BUILTIN_HLSL_FUNCTIONAL_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl" #include "nbl/builtin/hlsl/limits.hlsl" #include "nbl/builtin/hlsl/concepts/vector.hlsl" #include "nbl/builtin/hlsl/array_accessors.hlsl" diff --git a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl index 1549ce1990..69f932be06 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/core.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/core.hlsl @@ -16,6 +16,11 @@ namespace nbl { namespace hlsl { +// TODO: rename this namespace to `glsl_compat` +// The functions in this namespace are meant for porting GLSL shaders with minimal changes. +// Many of them are also handy utilities (default scopes and memory semantics, signed/unsigned/float dispatch, 16-bit emulation, +// a C++ path) and get called all over the codebase because they are easier to use than the raw `spirv::` intrinsics. +// Avoid calling them from code that isn't a GLSL port; those utilities need a home outside the `glsl` namespace. namespace glsl { @@ -260,7 +265,11 @@ uint32_t gl_InstanceIndex() {return spirv::InstanceIndex;} // TODO: Extemely annoying that HLSL doesn't have references, so we can't transparently alias the variables as `const&` :( uint32_t3 gl_NumWorkGroups() {return spirv::NumWorkGroups;} -// TODO: DXC BUG prevents us from defining this! +// Not a DXC bug: SPIR-V only lets a shader declare its workgroup size, there is nothing to read back. +// `[numthreads]` emits the `LocalSize` execution mode, whose operands are literals that no instruction can reference. +// `LocalSizeId` and the `WorkgroupSize` builtin take ids instead, but those must be constants (`WorkgroupSize` on an input +// variable is invalid), so "reading" them just means reusing the constant that declared the size in the first place. +// Hence the user defines this, returning the same compile-time constant they pass to `[numthreads]`. uint32_t3 gl_WorkGroupSize(); uint32_t3 gl_WorkGroupID() {return spirv::WorkgroupId;} uint32_t3 gl_LocalInvocationID() {return spirv::LocalInvocationId;} @@ -369,6 +378,8 @@ struct bitfieldExtract T bitfieldInsert(T base, T insert, uint32_t offset, uint32_t bits) { @@ -484,6 +495,7 @@ struct notEqual_helper inline vector::Dimension> equal(NBL_CONST_REF_ARG(T) x, NBL_CONST_REF_ARG(T) y) { diff --git a/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl b/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl index 148a61b540..15a015940e 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl @@ -13,6 +13,8 @@ namespace hlsl namespace glsl { +// TODO: these are used as general utilities outside GLSL ports (one name per operation for every type), +// they need a home outside the `glsl` namespace, see the note above `namespace glsl` in glsl_compat/core.hlsl // TODO: @Hazardu this will need touching up in the impl when the `spirv::groupXXX` will change names to a predictable pattern from SPIR-V // TODO: Furthermore you'll need `bitfieldExtract`-like struct dispatcher to choose between int/float add/mul and sint/uint/float min/max template diff --git a/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl b/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl index 2504a34480..20533f0c07 100644 --- a/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl +++ b/include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl @@ -17,6 +17,7 @@ namespace glsl // TODO: Extemely annoying that HLSL doesn't have referencies, so we can't transparently alias the variables as `const&` :( // NOTE: These are not `uint16_t` even though they could be, because IIUtSC they're `uint32_t` in SPIR-V uint32_t gl_SubgroupSize() {return spirv::SubgroupSize;} +// TODO: not a GLSL builtin, move it out of the `glsl` namespace to a better place uint32_t gl_SubgroupSizeLog2() {return firstbithigh(spirv::SubgroupSize);} uint32_t gl_SubgroupInvocationID() {return spirv::SubgroupLocalInvocationId;} diff --git a/include/nbl/builtin/hlsl/memory_accessor.hlsl b/include/nbl/builtin/hlsl/memory_accessor.hlsl index 2194b1e917..f31a9e8db5 100644 --- a/include/nbl/builtin/hlsl/memory_accessor.hlsl +++ b/include/nbl/builtin/hlsl/memory_accessor.hlsl @@ -4,7 +4,6 @@ #ifndef _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_ #define _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" #include "nbl/builtin/hlsl/member_test_macros.hlsl" // weird namespace placing, see the comment where the macro is defined diff --git a/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl b/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl index 577b69fc3c..55e7f78a25 100644 --- a/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl +++ b/include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl @@ -1,5 +1,4 @@ #include "nbl/builtin/hlsl/cpp_compat.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" #include "nbl/builtin/hlsl/workgroup/basic.hlsl" #include "nbl/builtin/hlsl/workgroup/arithmetic.hlsl" #include "nbl/builtin/hlsl/device_capabilities_traits.hlsl" diff --git a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl index ad2a2335ca..3364601c21 100644 --- a/include/nbl/builtin/hlsl/scan/chained_scan.hlsl +++ b/include/nbl/builtin/hlsl/scan/chained_scan.hlsl @@ -52,7 +52,7 @@ struct WorkgroupDataProxy void workgroupExecutionAndMemoryBarrier() { - glsl::barrier(); + spirv::controlBarrier(spv::ScopeWorkgroup, spv::ScopeWorkgroup, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask); //glsl::memoryBarrierShared(); implied by the above } @@ -106,7 +106,7 @@ struct Scan // should be the last element of the last workgroup thread // don't know what it's like if virtual workgroup size doesn't divide by workgroup size exactly - const uint32_t lastInvocIx = glsl::gl_SubgroupSize() * glsl::gl_NumSubgroups() - 1u; + const uint32_t lastInvocIx = spirv::SubgroupSize * spirv::NumSubgroups - 1u; scalar_t lastElem; if (invocIx == lastInvocIx) lastElem = wgDataAccessor.preloaded[wg_data_proxy_t::PreloadedDataCount-1u][Config::ItemsPerInvocation_0-1u]; diff --git a/include/nbl/builtin/hlsl/sort/counting.hlsl b/include/nbl/builtin/hlsl/sort/counting.hlsl index 1cd916ccc8..f5ecd827aa 100644 --- a/include/nbl/builtin/hlsl/sort/counting.hlsl +++ b/include/nbl/builtin/hlsl/sort/counting.hlsl @@ -131,7 +131,7 @@ struct counting sdata.workgroupExecutionAndMemoryBarrier(); uint32_t tid = workgroup::SubgroupContiguousIndex(); - const uint32_t shift = glsl::gl_SubgroupSize() * workGroupIndex; + const uint32_t shift = spirv::SubgroupSize * workGroupIndex; for (uint32_t vtid=tid; vtid must be a constant for WorkgroupSize"), see `glsl::gl_WorkGroupSize` //[[vk::ext_builtin_input(spv::BuiltInWorkgroupSize)]] //static const uint32_t3 WorkgroupSize; [[vk::ext_builtin_input(spv::BuiltInWorkgroupId)]] diff --git a/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl b/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl index ca5dd51dad..ea1e7197df 100644 --- a/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/arithmetic_portability_impl.hlsl @@ -5,6 +5,7 @@ #define _NBL_BUILTIN_HLSL_SUBGROUP_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ +// TODO: `glsl::subgroupShuffleUp` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" #include "nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl" @@ -94,8 +95,8 @@ struct inclusive_scan static type_t __call(type_t value) { Binop op; - const uint subgroupInvocation = glsl::gl_SubgroupInvocationID(); - const uint halfSubgroupSize = glsl::gl_SubgroupSize() >> 1u; + const uint subgroupInvocation = spirv::SubgroupLocalInvocationId; + const uint halfSubgroupSize = spirv::SubgroupSize >> 1u; type_t rhs = glsl::subgroupShuffleUp(value, 1u); // all invocations must execute the shuffle, even if we don't apply the op() to all of them value = op(value, subgroupInvocation<1u ? Binop::identity:rhs); @@ -121,7 +122,7 @@ struct exclusive_scan // can't risk getting short-circuited, need to store to a var type_t left = glsl::subgroupShuffleUp(value,1); // the first invocation doesn't have anything in its left so we set to the binop's identity value for exlusive scan - return bool(glsl::gl_SubgroupInvocationID()) ? left:Binop::identity; + return bool(spirv::SubgroupLocalInvocationId) ? left:Binop::identity; } }; diff --git a/include/nbl/builtin/hlsl/subgroup/ballot.hlsl b/include/nbl/builtin/hlsl/subgroup/ballot.hlsl index 4c9a817415..196a136891 100644 --- a/include/nbl/builtin/hlsl/subgroup/ballot.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/ballot.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP_BALLOT_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP_BALLOT_INCLUDED_ +// TODO: `glsl::subgroupBallot`, `glsl::subgroupBallotFindMSB`, `glsl::subgroupBroadcast` and `glsl::subgroupBroadcastFirst` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" #include "nbl/builtin/hlsl/subgroup/basic.hlsl" @@ -24,7 +25,7 @@ uint32_t LastSubgroupInvocation() bool ElectLast() { - return glsl::gl_SubgroupInvocationID()==LastSubgroupInvocation(); + return spirv::SubgroupLocalInvocationId==LastSubgroupInvocation(); } template @@ -34,7 +35,7 @@ T BroadcastLast(T value) } uint32_t ElectedSubgroupInvocationID() { - return glsl::subgroupBroadcastFirst(glsl::gl_SubgroupInvocationID()); + return glsl::subgroupBroadcastFirst(spirv::SubgroupLocalInvocationId); } } diff --git a/include/nbl/builtin/hlsl/subgroup/basic.hlsl b/include/nbl/builtin/hlsl/subgroup/basic.hlsl index b39c176e8e..de8d12d6c3 100644 --- a/include/nbl/builtin/hlsl/subgroup/basic.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/basic.hlsl @@ -4,8 +4,6 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP_BASIC_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP_BASIC_INCLUDED_ -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" - namespace nbl { namespace hlsl diff --git a/include/nbl/builtin/hlsl/subgroup/fft.hlsl b/include/nbl/builtin/hlsl/subgroup/fft.hlsl index f12f3e72e1..7c306f91df 100644 --- a/include/nbl/builtin/hlsl/subgroup/fft.hlsl +++ b/include/nbl/builtin/hlsl/subgroup/fft.hlsl @@ -2,7 +2,8 @@ #define _NBL_BUILTIN_HLSL_SUBGROUP_FFT_INCLUDED_ #include "nbl/builtin/hlsl/fft/common.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/subgroup_basic.hlsl" +// TODO: `glsl::subgroupShuffleXor` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" namespace nbl @@ -26,7 +27,7 @@ struct FFT { static void FFT_loop(uint32_t stride, NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const bool topHalf = bool(glsl::gl_SubgroupInvocationID() & stride); + const bool topHalf = bool(spirv::SubgroupLocalInvocationId & stride); const vector toTrade = topHalf ? vector (lo.real(), lo.imag()) : vector (hi.real(), hi.imag()); const vector exchanged = glsl::subgroupShuffleXor< vector > (toTrade, stride); if (topHalf) @@ -40,15 +41,15 @@ struct FFT hi.imag(exchanged.y); } // Get twiddle with k = subgroupInvocation mod stride, halfN = stride - fft::DIF::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID() & (stride - 1), stride), lo, hi); + fft::DIF::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId & (stride - 1), stride), lo, hi); } static void __call(NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const uint32_t subgroupSize = glsl::gl_SubgroupSize(); //This is N/2 + const uint32_t subgroupSize = spirv::SubgroupSize; //This is N/2 // special first iteration - fft::DIF::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID(), subgroupSize), lo, hi); + fft::DIF::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId, subgroupSize), lo, hi); // Decimation in Frequency [unroll] @@ -66,9 +67,9 @@ struct FFT static void FFT_loop(uint32_t stride, NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { // Get twiddle with k = subgroupInvocation mod stride, halfN = stride - fft::DIT::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID() & (stride - 1), stride), lo, hi); + fft::DIT::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId & (stride - 1), stride), lo, hi); - const bool topHalf = bool(glsl::gl_SubgroupInvocationID() & stride); + const bool topHalf = bool(spirv::SubgroupLocalInvocationId & stride); const vector toTrade = topHalf ? vector (lo.real(), lo.imag()) : vector (hi.real(), hi.imag()); const vector exchanged = glsl::subgroupShuffleXor< vector > (toTrade, stride); if (topHalf) @@ -85,7 +86,7 @@ struct FFT static void __call(NBL_REF_ARG(complex_t) lo, NBL_REF_ARG(complex_t) hi) { - const uint32_t subgroupSize = glsl::gl_SubgroupSize(); //This is N/2 + const uint32_t subgroupSize = spirv::SubgroupSize; //This is N/2 const uint32_t doubleSubgroupSize = subgroupSize << 1; //This is N // Decimation in Time @@ -94,7 +95,7 @@ struct FFT FFT_loop(stride, lo, hi); // special last iteration - fft::DIT::radix2(fft::twiddle(glsl::gl_SubgroupInvocationID(), subgroupSize), lo, hi); + fft::DIT::radix2(fft::twiddle(spirv::SubgroupLocalInvocationId, subgroupSize), lo, hi); divides_assign< complex_t > divAss; divAss(lo, Scalar(doubleSubgroupSize)); divAss(hi, Scalar(doubleSubgroupSize)); diff --git a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl index 18f20c3cfc..1d7821e47a 100644 --- a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP2_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP2_ARITHMETIC_PORTABILITY_IMPL_INCLUDED_ +// TODO: `glsl::subgroupShuffleUp` and `glsl::subgroupShuffleXor` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_shuffle.hlsl" #include "nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl" @@ -148,7 +149,7 @@ struct inclusive_scan_impl { inclusive_scan_impl retval; retval.value = _value; - retval.subgroupInvocation = glsl::gl_SubgroupInvocationID(); + retval.subgroupInvocation = spirv::SubgroupLocalInvocationId; return retval; } @@ -200,7 +201,7 @@ struct exclusive_scan // sync up each subgroup invocation so it runs in lockstep spirv::controlBarrier(spv::ScopeSubgroup, spv::ScopeSubgroup, spv::MemorySemanticsMaskNone); - scalar_t left = hlsl::mix(binop_t::identity, glsl::subgroupShuffleUp(value,1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left = hlsl::mix(binop_t::identity, glsl::subgroupShuffleUp(value,1), bool(spirv::SubgroupLocalInvocationId)); return inclusive_scan::__call(left); } }; diff --git a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl index 3b511126b4..2b3a61ea32 100644 --- a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_SUBGROUP2_BALLOT_INCLUDED_ #define _NBL_BUILTIN_HLSL_SUBGROUP2_BALLOT_INCLUDED_ +// TODO: `glsl::subgroupBallot` and `glsl::subgroupBallotFindMSB` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" namespace nbl @@ -17,14 +18,14 @@ template uint32_t LastSubgroupInvocation() { if (AssumeAllActive) - return glsl::gl_SubgroupSize()-1; + return spirv::SubgroupSize-1; else return glsl::subgroupBallotFindMSB(glsl::subgroupBallot(true)); } bool ElectLast() { - return glsl::gl_SubgroupInvocationID()==LastSubgroupInvocation(); + return spirv::SubgroupLocalInvocationId==LastSubgroupInvocation(); } template diff --git a/include/nbl/builtin/hlsl/surface_transform.h b/include/nbl/builtin/hlsl/surface_transform.h index 0c9b54ce5a..88f4f9ddc2 100644 --- a/include/nbl/builtin/hlsl/surface_transform.h +++ b/include/nbl/builtin/hlsl/surface_transform.h @@ -5,7 +5,6 @@ #define _NBL_BUILTIN_HLSL_SURFACE_TRANSFORM_INCLUDED_ #include #include -#include namespace nbl { diff --git a/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl b/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl index f8c5c75cb0..a57788acfd 100644 --- a/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/arithmetic.hlsl @@ -115,7 +115,7 @@ uint16_t ballotScanBitCount(NBL_REF_ARG(BallotAccessor) ballotAccessor, NBL_REF_ arithmeticAccessor.set(subgroupIndex,count); arithmeticAccessor.workgroupExecutionAndMemoryBarrier(); arithmeticAccessor.get(bitfieldIndex, count); - return uint16_t(countbits(localBitfield&(Exclusive ? glsl::gl_SubgroupLtMask():glsl::gl_SubgroupLeMask())[getDWORD(uint16_t(glsl::gl_SubgroupInvocationID()))])+count); + return uint16_t(countbits(localBitfield&(Exclusive ? spirv::BuiltInSubgroupLtMask : spirv::BuiltInSubgroupLeMask)[getDWORD(uint16_t(spirv::SubgroupLocalInvocationId))])+count); } } diff --git a/include/nbl/builtin/hlsl/workgroup/ballot.hlsl b/include/nbl/builtin/hlsl/workgroup/ballot.hlsl index bcfdde8c4b..c6cecd0e82 100644 --- a/include/nbl/builtin/hlsl/workgroup/ballot.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/ballot.hlsl @@ -8,6 +8,8 @@ #include "nbl/builtin/hlsl/workgroup/basic.hlsl" #include "nbl/builtin/hlsl/subgroup/arithmetic_portability.hlsl" +// TODO: `glsl::subgroupElect` and `glsl::subgroupBallot` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace + namespace nbl { namespace hlsl @@ -43,7 +45,7 @@ struct ballot if (glsl::subgroupElect()) { const uint16_t SubgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); const uint16_t shift = (subgroupID<>(5-SubgroupSizeLog2),subgroupBallot[0]< static void __call(const uint32_t4 subgroupBallot, const uint16_t subgroupInvocation, NBL_REF_ARG(Accessor) accessor) { const uint16_t SubgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); const uint16_t destIx = subgroupID<<(SubgroupSizeLog2-5); const uint16_t UsefulComponents = uint16_t(0x1u)<<(SubgroupSizeLog2-5); @@ -89,7 +91,7 @@ template void ballot(const bool value, NBL_REF_ARG(Accessor) accessor) { const uint32_t4 bitfield = glsl::subgroupBallot(value); - const uint16_t subgroupInvocation = uint16_t(glsl::gl_SubgroupInvocationID()); + const uint16_t subgroupInvocation = uint16_t(spirv::SubgroupLocalInvocationId); if (glsl::gl_SubgroupSizeLog2()<4) impl::ballot::template __call(bitfield,subgroupInvocation,accessor); diff --git a/include/nbl/builtin/hlsl/workgroup/basic.hlsl b/include/nbl/builtin/hlsl/workgroup/basic.hlsl index 3467b46407..b93dae11be 100644 --- a/include/nbl/builtin/hlsl/workgroup/basic.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/basic.hlsl @@ -4,6 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_WORKGROUP_BASIC_INCLUDED_ #define _NBL_BUILTIN_HLSL_WORKGROUP_BASIC_INCLUDED_ +// TODO: `glsl::subgroupBroadcastFirst` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace #include "nbl/builtin/hlsl/glsl_compat/subgroup_ballot.hlsl" //! all functions must be called in uniform control flow (all workgroup invocations active) @@ -25,14 +26,14 @@ uint16_t Volume() uint16_t SubgroupContiguousIndex() { - const uint16_t retval = (uint16_t(glsl::gl_SubgroupID())<(otherElementIndex, workgroup::SubgroupContiguousIndex(), 0, uint32_t(WorkgroupSizeLog2)); + const uint32_t mirrorGlobalIndex = spirv::bitFieldInsert(otherElementIndex, workgroup::SubgroupContiguousIndex(), 0, uint32_t(WorkgroupSizeLog2)); const uint32_t otherThreadID = otherElementIndex & (WorkgroupSize - 1); const NablaMirrorGlobalInfo info = { otherThreadID, mirrorGlobalIndex }; return info; @@ -339,7 +339,7 @@ struct FFT, device accessor.get(hiIx, hi); // If for some reason you're running a small FFT, skip all the bigger-than-subgroup steps - if (WorkgroupSize > glsl::gl_SubgroupSize()) + if (WorkgroupSize > spirv::SubgroupSize) { // Set up the memory adaptor using adaptor_t = accessor_adaptors::StructureOfArrays; @@ -351,14 +351,14 @@ struct FFT, device // Run bigger steps until Subgroup-sized [unroll] - for (uint32_t stride = WorkgroupSize >> 1; stride > glsl::gl_SubgroupSize(); stride >>= 1) + for (uint32_t stride = WorkgroupSize >> 1; stride > spirv::SubgroupSize; stride >>= 1) { FFT_loop< adaptor_t >(stride, lo, hi, threadID, sharedmemAdaptor); sharedmemAdaptor.workgroupExecutionAndMemoryBarrier(); } // special last workgroup-shuffle - fft::impl::exchangeValues::__call(lo, hi, threadID, glsl::gl_SubgroupSize(), sharedmemAdaptor); + fft::impl::exchangeValues::__call(lo, hi, threadID, spirv::SubgroupSize, sharedmemAdaptor); // Remember to update the accessor's state sharedmemAccessor = sharedmemAdaptor.accessor; @@ -408,7 +408,7 @@ struct FFT, device_ subgroup::FFT::__call(lo, hi); // If for some reason you're running a small FFT, skip all the bigger-than-subgroup steps - if (WorkgroupSize > glsl::gl_SubgroupSize()) + if (WorkgroupSize > spirv::SubgroupSize) { // Set up the memory adaptor using adaptor_t = accessor_adaptors::StructureOfArrays; @@ -416,11 +416,11 @@ struct FFT, device_ sharedmemAdaptor.accessor = sharedmemAccessor; // special first workgroup-shuffle - fft::impl::exchangeValues::__call(lo, hi, threadID, glsl::gl_SubgroupSize(), sharedmemAdaptor); + fft::impl::exchangeValues::__call(lo, hi, threadID, spirv::SubgroupSize, sharedmemAdaptor); // The bigger steps [unroll] - for (uint32_t stride = glsl::gl_SubgroupSize() << 1; stride < WorkgroupSize; stride <<= 1) + for (uint32_t stride = spirv::SubgroupSize << 1; stride < WorkgroupSize; stride <<= 1) { // Order of waiting for shared mem writes is also reversed here, since the shuffle came earlier sharedmemAdaptor.workgroupExecutionAndMemoryBarrier(); @@ -430,8 +430,8 @@ struct FFT, device_ // special last iteration hlsl::fft::DIT::radix2(hlsl::fft::twiddle(threadID, WorkgroupSize), lo, hi); divides_assign< complex_t > divAss; - divAss(lo, Scalar(WorkgroupSize / glsl::gl_SubgroupSize())); - divAss(hi, Scalar(WorkgroupSize / glsl::gl_SubgroupSize())); + divAss(lo, Scalar(WorkgroupSize / spirv::SubgroupSize)); + divAss(hi, Scalar(WorkgroupSize / spirv::SubgroupSize)); // Remember to update the accessor's state sharedmemAccessor = sharedmemAdaptor.accessor; diff --git a/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl b/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl index b52d083733..8a3bb11241 100644 --- a/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/shared_scan.hlsl @@ -6,7 +6,8 @@ #include "nbl/builtin/hlsl/cpp_compat.hlsl" #include "nbl/builtin/hlsl/workgroup/broadcast.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +// TODO: `glsl::subgroupElect` and `glsl::subgroupShuffleUp` are only used for their default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace +#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" #include "nbl/builtin/hlsl/subgroup/arithmetic_portability.hlsl" namespace nbl @@ -27,7 +28,7 @@ struct reduce void __call(NBL_CONST_REF_ARG(type_t) value, NBL_REF_ARG(Accessor) scratchAccessor) { const uint16_t lastInvocation = ItemCount-_static_cast(1); - const uint16_t subgroupMask = _static_cast(glsl::gl_SubgroupSize()-1u); + const uint16_t subgroupMask = _static_cast(spirv::SubgroupSize-1u); subgroup::inclusive_scan subgroupOp; @@ -39,7 +40,7 @@ struct reduce type_t scan = firstLevelScan; // could use ElectLast() but we can optimize for full workgroups here - const bool isLastSubgroupInvocation = uint16_t(glsl::gl_SubgroupInvocationID())==subgroupMask; + const bool isLastSubgroupInvocation = uint16_t(spirv::SubgroupLocalInvocationId)==subgroupMask; // Since we are scanning the RESULT of the initial scan (which paired one input per subgroup invocation) // every group of gl_SubgroupSz invocations has been coallesced into 1 result value. This means that the results of @@ -47,7 +48,7 @@ struct reduce // Consequently, those first gl_SubgroupSz^2 invocations will store their results on gl_SubgroupSz scratch slots // and the next level will follow the same + the previous as an `offset`. - const uint16_t loadStoreIndexDiff = scanLoadIndex-_static_cast(glsl::gl_SubgroupID()); + const uint16_t loadStoreIndexDiff = scanLoadIndex-_static_cast(spirv::SubgroupId); // to cancel out the index shift on the first iteration if (lastInvocationInLevel>subgroupMask) @@ -91,13 +92,13 @@ struct scan// : reduce https://github.com/microsoft/DirectXShad { __base.template __call(value,scratchAccessor); - const uint16_t subgroupID = uint16_t(glsl::gl_SubgroupID()); + const uint16_t subgroupID = uint16_t(spirv::SubgroupId); // abuse integer wraparound to map 0 to 0xffffu const uint16_t prevSubgroupID = subgroupID-_static_cast(1); // important check to prevent weird `firstbithigh` overlflows const uint16_t lastInvocation = ItemCount-_static_cast(1); - if(lastInvocation>=uint16_t(glsl::gl_SubgroupSize())) + if(lastInvocation>=uint16_t(spirv::SubgroupSize)) { const uint16_t subgroupSizeLog2 = uint16_t(glsl::gl_SubgroupSizeLog2()); // different than Upsweep cause we need to translate high level inclusive scans into exclusive on the fly, so we get the value of the subgroup behind our own in each level diff --git a/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl b/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl index 22c93ce193..3cc3cfc1d1 100644 --- a/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl +++ b/include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl @@ -79,7 +79,7 @@ struct ArithmeticConfiguration #ifdef __HLSL_VERSION static bool electLast() { - return glsl::gl_SubgroupInvocationID()==SubgroupSize-1; + return spirv::SubgroupLocalInvocationId==SubgroupSize-1; } #endif diff --git a/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl b/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl index 613ba0ddca..f7a45d4e2f 100644 --- a/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl +++ b/include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl @@ -5,13 +5,15 @@ #define _NBL_BUILTIN_HLSL_WORKGROUP2_SHARED_SCAN_INCLUDED_ #include "nbl/builtin/hlsl/workgroup/broadcast.hlsl" -#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" +#include "nbl/builtin/hlsl/spirv_intrinsics/subgroup_basic.hlsl" #include "nbl/builtin/hlsl/subgroup2/ballot.hlsl" #include "nbl/builtin/hlsl/subgroup2/arithmetic_params.hlsl" #include "nbl/builtin/hlsl/subgroup2/arithmetic_portability.hlsl" #include "nbl/builtin/hlsl/mpl.hlsl" #include "nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl" +// TODO: `glsl::subgroupShuffleUp` is only used for its default `spv::ScopeSubgroup`, drop this dependency on the `glsl` namespace once the subgroup utilities move to a better namespace + namespace nbl { namespace hlsl @@ -44,7 +46,7 @@ struct reduce subgroup2::reduction reduction; vector_t value; - dataAccessor.template get(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template get(uint16_t(spirv::SubgroupLocalInvocationId), value); return reduction(value); } }; @@ -63,7 +65,7 @@ struct scan using params_t = subgroup2::ArithmeticParams; vector_t value; - dataAccessor.template get(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template get(uint16_t(spirv::SubgroupLocalInvocationId), value); if (Exclusive) { subgroup2::exclusive_scan excl_scan; @@ -74,7 +76,7 @@ struct scan subgroup2::inclusive_scan incl_scan; value = incl_scan(value); } - dataAccessor.template set(uint16_t(glsl::gl_SubgroupInvocationID()), value); + dataAccessor.template set(uint16_t(spirv::SubgroupLocalInvocationId), value); } }; @@ -102,7 +104,7 @@ struct reduce_level0 scan_local = reduction0(scan_local); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId), idx); scratchAccessor.template set(bankedIndex, scan_local[Config::ItemsPerInvocation_0-1]); // set last element of subgroup scan (reduction) to level 1 scan } } @@ -134,7 +136,7 @@ struct scan_level0 dataAccessor.template set(idx * Config::WorkgroupSize + virtualInvocationIndex, value); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId), idx); scratchAccessor.template set(bankedIndex, value[Config::ItemsPerInvocation_0-1]); // set last element of subgroup scan (reduction) to level 1 scan } } @@ -162,7 +164,7 @@ struct reduce const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::reduction reduction1; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv1_t lv1_val; [unroll] @@ -200,7 +202,7 @@ struct scan const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::inclusive_scan inclusiveScan1; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv1_t lv1_val; [unroll] @@ -220,13 +222,13 @@ struct scan vector_lv0_t value; dataAccessor.template get(idx * Config::WorkgroupSize + virtualInvocationIndex, value); - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()-1u), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId-1u), idx); scalar_t left = BinOp::identity; - if (idx != 0 || glsl::gl_SubgroupID() != 0) + if (idx != 0 || spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex,left); if (Exclusive) { - scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(spirv::SubgroupLocalInvocationId)); [unroll] for (uint16_t i = Config::ItemsPerInvocation_0-1; i > 0; i--) value[i] = binop(left, value[i-1]); @@ -265,7 +267,7 @@ struct reduce const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::reduction reduction1; - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -274,7 +276,7 @@ struct reduce lv1_val = reduction1(lv1_val); if (Config::electLast()) { - const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(glsl::gl_SubgroupID())); + const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(spirv::SubgroupId)); scratchAccessor.template set(bankedIndex, lv1_val[Config::ItemsPerInvocation_1-1]); } } @@ -282,7 +284,7 @@ struct reduce // level 2 scan subgroup2::reduction reduction2; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { vector_lv2_t lv2_val; [unroll] @@ -321,7 +323,7 @@ struct scan const uint16_t invocationIndex = workgroup::SubgroupContiguousIndex(); // level 1 scan subgroup2::inclusive_scan inclusiveScan1; - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -336,7 +338,7 @@ struct scan // level 2 scan subgroup2::inclusive_scan inclusiveScan2; - if (glsl::gl_SubgroupID() == 0) + if (spirv::SubgroupId == 0) { const uint16_t lastChannel = Config::ItemsPerInvocation_1 - uint16_t(1u); vector_lv2_t lv2_val; @@ -355,7 +357,7 @@ struct scan scratchAccessor.workgroupExecutionAndMemoryBarrier(); // combine with level 1 - if (glsl::gl_SubgroupID() < Config::LevelInputCount_2) + if (spirv::SubgroupId < Config::LevelInputCount_2) { vector_lv1_t lv1_val; [unroll] @@ -363,8 +365,8 @@ struct scan scratchAccessor.template get(Config::template sharedLoadIndex<1>(invocationIndex, i), lv1_val[i]); scalar_t lv2_scan = BinOp::identity; - const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(glsl::gl_SubgroupID()-1u)); - if (glsl::gl_SubgroupID() != 0) + const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(spirv::SubgroupId-1u)); + if (spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex, lv2_scan); [unroll] @@ -380,13 +382,13 @@ struct scan vector_lv0_t value; dataAccessor.template get(idx * Config::WorkgroupSize + virtualInvocationIndex, value); - const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(glsl::gl_SubgroupID()-1u), idx); + const uint16_t bankedIndex = Config::template sharedStoreIndexFromVirtualIndex<1>(uint16_t(spirv::SubgroupId-1u), idx); scalar_t left = BinOp::identity; - if (idx != 0 || glsl::gl_SubgroupID() != 0) + if (idx != 0 || spirv::SubgroupId != 0) scratchAccessor.template get(bankedIndex,left); if (Exclusive) { - scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(glsl::gl_SubgroupInvocationID())); + scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp(value[Config::ItemsPerInvocation_0-1],1), bool(spirv::SubgroupLocalInvocationId)); [unroll] for (uint16_t i = Config::ItemsPerInvocation_0-1; i > 0; i--) value[i] = binop(left, value[i-1]); From f7d10b0cd1eec1cf1ec48f74aec7437a0dcf2111 Mon Sep 17 00:00:00 2001 From: Erfan Ahmadi Date: Thu, 22 Oct 2026 17:37:18 +0400 Subject: [PATCH 29/31] updated examples_tests --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 105f94003a..9a380ef63b 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 105f94003a2b11c7ab3b349d6a8cb6474ca5cbff +Subproject commit 9a380ef63b57a19d0108d324f7c49c2e57315e5d From 1ea808d01e4c28a2149bcd405574b800c8fcee17 Mon Sep 17 00:00:00 2001 From: Erfan Ahmadi Date: Fri, 23 Oct 2026 16:58:13 +0400 Subject: [PATCH 30/31] comment about SubgroupContiguousIndex --- include/nbl/builtin/hlsl/workgroup/basic.hlsl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/nbl/builtin/hlsl/workgroup/basic.hlsl b/include/nbl/builtin/hlsl/workgroup/basic.hlsl index b93dae11be..79af5f74ee 100644 --- a/include/nbl/builtin/hlsl/workgroup/basic.hlsl +++ b/include/nbl/builtin/hlsl/workgroup/basic.hlsl @@ -23,7 +23,10 @@ uint16_t Volume() const uint16_t3 dims = uint16_t3(glsl::gl_WorkGroupSize()); return dims.x*dims.y*dims.z; } - + +// Use this instead of `LocalInvocationIndex`: that doesn't guarantee indices 0..31 come from the first subgroup (with a SubgroupSize of 32), this function does. +// Vulkan defines no relationship between `SubgroupLocalInvocationId` and `LocalInvocationIndex`, and suggests this formula instead, +// valid only with full subgroups: https://docs.vulkan.org/refpages/latest/refpages/source/SubgroupLocalInvocationId.html uint16_t SubgroupContiguousIndex() { const uint16_t retval = (uint16_t(spirv::SubgroupId)< Date: Fri, 23 Oct 2026 18:20:21 +0400 Subject: [PATCH 31/31] explainer comments on subgroup2, todo for removing glsl:: dependency, and new README.md for subgroup2 stuff --- include/nbl/builtin/hlsl/subgroup2/README.md | 130 ++++++++++++++++++ .../arithmetic_portability_impl.hlsl | 14 ++ .../nbl/builtin/hlsl/subgroup2/ballot.hlsl | 1 + 3 files changed, 145 insertions(+) create mode 100644 include/nbl/builtin/hlsl/subgroup2/README.md diff --git a/include/nbl/builtin/hlsl/subgroup2/README.md b/include/nbl/builtin/hlsl/subgroup2/README.md new file mode 100644 index 0000000000..d46bc03d81 --- /dev/null +++ b/include/nbl/builtin/hlsl/subgroup2/README.md @@ -0,0 +1,130 @@ +# subgroup2 arithmetic + +Not every device supports subgroup arithmetic (`VK_SUBGROUP_FEATURE_ARITHMETIC_BIT`). So `subgroup2::reduction`, `subgroup2::inclusive_scan` and `subgroup2::exclusive_scan` either use the native subgroup instructions or emulate them with subgroup shuffles. The choice is made at compile time from `device_capabilities::shaderSubgroupArithmetic`, which you set from the device's limits on the host. + +Both paths combine values across one subgroup, and the rest of the configuration comes from the `ArithmeticParams` you pass in. + +- [ArithmeticParams](#arithmeticparams) +- [What ItemsPerInvocation is](#what-itemsperinvocation-is) +- [How the specializations are chosen](#how-the-specializations-are-chosen) +- [Seeing it in SPIR-V](#seeing-it-in-spir-v) + +## ArithmeticParams + +```cpp +using config_t = subgroup2::Configuration<5>; // SubgroupSize 32 +using params_t = subgroup2::ArithmeticParams, 1, device_capabilities>; +subgroup2::inclusive_scan scan; +const params_t::type_t result = scan(value); +``` + +| `ArithmeticParams` member | Comes from | +|---|---| +| `config_t` | `Configuration`. Must match the subgroup size the pipeline runs with. | +| `binop_t`, `scalar_t` | the binop, e.g. `plus`, and its `type_t` | +| `type_t` | `vector`: each invocation holds `ItemsPerInvocation` consecutive elements | +| `UseNativeIntrinsics` | `device_capabilities::shaderSubgroupArithmetic`, `false` when the member is missing or `device_capabilities` is `void` | + +## What ItemsPerInvocation is + +**Each thread holds N consecutive elements in one `vector`, instead of one element. That's all it means.** (N is `ItemsPerInvocation`.) + +With a subgroup of 32 and `ItemsPerInvocation = 2`, one subgroup covers 32 × 2 = 64 elements, and lane `k` holds elements `2k` and `2k+1`: + +| lane | holds | element indices | +|---|---|---| +| 0 | `(e0, e1)` | 0, 1 | +| 1 | `(e2, e3)` | 2, 3 | +| ... | ... | ... | +| 31 | `(e62, e63)` | 62, 63 | + +The subgroup still has 32 lanes, and an operation still needs only one cross-lane step. The extra items are combined serially inside each invocation. + +For example, take an inclusive `plus` scan of 64 ones. The steps are the numbered comments in the [primary `inclusive_scan`](arithmetic_portability_impl.hlsl#L39-L72): + +| | lane 0 | lane 1 | lane 2 | ... | lane 31 | +|---|---|---|---|---|---| +| holds | (1, 1) | (1, 1) | (1, 1) | ... | (1, 1) | +| (1) own items scanned | (1, **2**) | (1, **2**) | (1, **2**) | ... | (1, **2**) | +| (2) subgroup exclusive scan of the totals | 0 | 2 | 4 | ... | 62 | +| (3) combined with every item | (1, 2) | (3, 4) | (5, 6) | ... | (63, 64) | + +`workgroup2::ArithmeticConfiguration`'s `ItemsPerInvocation_0` is this same value. It's the vector type workgroup data accessors read and write. + +## How the specializations are chosen + +The public structs in `arithmetic_portability.hlsl` inherit from `impl::`, defined in `arithmetic_portability_impl.hlsl`. **`ItemsPerInvocation` is what selects the primary template, not native support:** + +| `ItemsPerInvocation` | native | Selected | What it does | +|---|---|---|---| +| not 1 | either | primary template | Combines the invocation's own items serially, then makes one `ItemsPerInvocation = 1` call (the exclusive scan for both scans, the reduction for `reduction`) on each invocation's total. See [What ItemsPerInvocation is](#what-itemsperinvocation-is). | +| 1 | `true` | partial specialization on `BinOp`, written out by the `SPECIALIZE_ALL` macro for `bit_and`, `bit_or`, `bit_xor`, `plus`, `multiplies`, `minimum`, `maximum` | one `glsl::subgroup{Inclusive,Exclusive,}{And,Or,Xor,Add,Mul,Min,Max}` call | +| 1 | `false` | partial specialization on ``, for any binop | Emulation with `SizeLog2` shuffle steps. Inclusive scan: Hillis-Steele with `subgroupShuffleUp`. Exclusive scan: shift up by one lane, then inclusive. Reduction: `subgroupShuffleXor` butterfly. | + +So native support only decides which of the two `ItemsPerInvocation = 1` versions does the cross-lane step, whether it's called directly or from inside the primary template. + +A binop without a `SPECIALIZE_ALL` line has no `<..., 1, true>` specialization. With native on, even `ItemsPerInvocation = 1` falls back to the primary template, which then calls itself, and DXC rejects it with `recursive functions are not allowed`. Use custom binops with `shaderSubgroupArithmetic = false`. + +## Seeing it in SPIR-V + +The shader below compiles one inclusive `plus` scan over a subgroup of 32. Change the three macros to pick a different specialization. + +```cpp +#include "nbl/builtin/hlsl/functional.hlsl" +#include "nbl/builtin/hlsl/workgroup/basic.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_params.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_portability.hlsl" + +// change these to pick a different specialization +#ifndef SUBGROUP_SIZE_LOG2 +#define SUBGROUP_SIZE_LOG2 5 // subgroup of 32 +#endif +#ifndef ITEMS_PER_INVOCATION +#define ITEMS_PER_INVOCATION 1 +#endif +#ifndef NATIVE +#define NATIVE 1 +#endif + +struct device_capabilities +{ + NBL_CONSTEXPR_STATIC_INLINE bool shaderSubgroupArithmetic = bool(NATIVE); +}; + +using namespace nbl::hlsl; +using config_t = subgroup2::Configuration; +using params_t = subgroup2::ArithmeticParams, ITEMS_PER_INVOCATION, device_capabilities>; + +[[vk::binding(0,0)]] RWStructuredBuffer data; + +[numthreads(64,1,1)] +void main() +{ + const uint32_t ix = workgroup::SubgroupContiguousIndex(); + subgroup2::inclusive_scan scan; + data[ix] = scan(data[ix]); +} +``` + +``` +nsc -spirv -HV 202x -T cs_6_7 -E main -fspv-target-env=vulkan1.3 -enable-16bit-types -DNATIVE=1 -DITEMS_PER_INVOCATION=1 -Fc scan.spv scan.hlsl +``` + +On Compiler Explorer (the release `nsc`, not necessarily your branch): + +| Variant | Link | +|---|---| +| native, 1 item | https://godbolt.devsh.eu/z/Waoao1 | +| emulated, 1 item | https://godbolt.devsh.eu/z/Yexhc4 | +| native, 4 items | https://godbolt.devsh.eu/z/z5rbno | +| emulated, 4 items | https://godbolt.devsh.eu/z/Y67zWK | +| all four side by side | https://godbolt.devsh.eu/z/abM995 | + +What each variant compiles to, with SubgroupSize 32: + +| Variant | Subgroup instructions | `OpIAdd` | +|---|---|---| +| native, 1 item | 1 `OpGroupNonUniformIAdd InclusiveScan` | 1: the `SubgroupContiguousIndex` | +| emulated, 1 item | 5 × (`OpControlBarrier`, `OpGroupNonUniformShuffleUp`, `OpSelect`), one per shuffle step | 6: 5 steps + index | +| native, 4 items | 1 `OpGroupNonUniformIAdd ExclusiveScan` (the primary template asks for the exclusive scan) | 8: 3 to scan the 4 items + 4 to add the exclusive result + index | +| emulated, 4 items | 6 × (`OpControlBarrier`, `OpGroupNonUniformShuffleUp`, `OpSelect`): 1 to shift up a lane + 5 inclusive steps | 13: 3 + 5 + 4 + index | diff --git a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl index 1d7821e47a..b909db02f7 100644 --- a/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/arithmetic_portability_impl.hlsl @@ -36,6 +36,9 @@ template struct reduction; +// Primary templates, used when ItemsPerInvocation != 1: each invocation holds ItemsPerInvocation consecutive elements in a vector. +// They combine the invocation's own items serially, so a single `ItemsPerInvocation = 1` subgroup op (native or emulated, +// specialized below) is all that crosses lanes. // BinOp needed to specialize native template struct inclusive_scan @@ -50,14 +53,17 @@ struct inclusive_scan { binop_t binop; type_t retval; + // (1) inclusive scan of this invocation's own items retval[0] = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i-1], value[i]); + // (2) one subgroup exclusive scan of each invocation's total: everything held by the lanes before this one exclusive_scan_op_t op; scalar_t exclusive = op(retval[ItemsPerInvocation-1]); + // (3) combine that with every item [unroll] for (uint32_t i = 0; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i], exclusive); @@ -77,14 +83,17 @@ struct exclusive_scan { binop_t binop; type_t retval; + // (1) inclusive scan of this invocation's own items retval[0] = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval[i] = binop(retval[i-1], value[i]); + // (2) one subgroup exclusive scan of each invocation's total: everything held by the lanes before this one exclusive_scan_op_t op; scalar_t exclusive = op(retval[ItemsPerInvocation-1]); + // (3) shift the items up by one to make them exclusive, combining each with that; item 0 gets it as is [unroll] for (uint32_t i = ItemsPerInvocation-1; i > 0; i--) retval[i] = binop(exclusive,retval[i-1]); @@ -105,16 +114,21 @@ struct reduction { binop_t binop; op_t op; + // (1) reduce this invocation's own items scalar_t retval = value[0]; [unroll] for (uint32_t i = 1; i < ItemsPerInvocation; i++) retval = binop(retval, value[i]); + // (2) one subgroup reduction of each invocation's total return op(retval); } }; // specs for N=1 uses subgroup funcs +// TODO: drop the weird `glsl::` dependency, call `spirv::group*` directly with `spv::ScopeSubgroup` and the `GroupOperation`, or a utility outside `glsl`. +// `plus`, `multiplies` and the bitwise ops map 1:1; `minimum`/`maximum` need signed/unsigned/float dispatch (`spirv::groupSMin/UMin/FMin`), +// either as 3 constrained specializations each (primaries need `NBL_STRUCT_CONSTRAINABLE`) or a shared dispatch helper also used by `glsl::subgroupMin/Max`. // specialize native #define SPECIALIZE(NAME,BINOP,SUBGROUP_OP) template struct NAME,1,true> \ { \ diff --git a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl index 2b3a61ea32..74e2bfa4fb 100644 --- a/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl +++ b/include/nbl/builtin/hlsl/subgroup2/ballot.hlsl @@ -31,6 +31,7 @@ bool ElectLast() template struct Configuration { + // smallest type holding one bit per subgroup invocation, for ballot-style masks: `uint32_t1` up to a SubgroupSize of 32, `uint32_t2` for 64, `uint32_t4` for 128 using mask_t = conditional_t, uint32_t4>; NBL_CONSTEXPR_STATIC_INLINE uint16_t SizeLog2 = uint16_t(SubgroupSizeLog2);