Device-wide chained scan with lookback

include/nbl/builtin/hlsl/bda/bda_accessor.hlsl

@@ -4,7 +4,7 @@
 #ifndef _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_
 #define _NBL_BUILTIN_HLSL_BDA_ACCESSOR_INCLUDED_
 
-#include "nbl/builtin/hlsl/glsl_compat/core.hlsl"
+#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl"
 #include "nbl/builtin/hlsl/bda/__ptr.hlsl"
 
 namespace nbl 
@@ -66,15 +66,15 @@ struct BdaAccessor : impl::BdaAccessorBase
     atomicAdd(const uint64_t index, const T value)
     {
         bda::__ptr<T> target = ptr + index;
-        return glsl::atomicAdd(target.template deref().ptr.value, value);
+        return spirv::atomicIAdd<T>(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
     }
 
     template<typename S = T>
     enable_if_t<is_same_v<S,T> && is_integral<T>::value && (sizeof(T) == 4 || sizeof(T) == 8), T>
     atomicSub(const uint64_t index, const T value)
     {
         bda::__ptr<T> target = ptr + index;
-        return glsl::atomicSub(target.template deref().ptr.value, value);
+        return spirv::atomicISub<T>(target.template deref().ptr.value, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
     }
 
     bda::__ptr<T> ptr;

include/nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl

@@ -9,7 +9,7 @@
 #ifndef _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_
 #define _NBL_BUILTIN_HLSL_LEGACY_BDA_ACCESSOR_INCLUDED_
 
-#include "nbl/builtin/hlsl/glsl_compat/core.hlsl"
+#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl"
 #include "nbl/builtin/hlsl/bda/__ptr.hlsl"
 
 

include/nbl/builtin/hlsl/blit/default_blit.comp.hlsl

@@ -67,7 +67,7 @@ void main()
 	OutImgAccessor outImgA;
 	outImgA.descIx = pc.outputDescIx;
 
-	const uint16_t3 wgID = _static_cast<uint16_t3>(glsl::gl_WorkGroupID());
+	const uint16_t3 wgID = _static_cast<uint16_t3>(spirv::WorkgroupId);
 	const uint16_t3 baseCoord = pc.perWG.getOutputBaseCoord(wgID);
 	// TODO: If and when someone can be bothered, change the blit api to compile a pipeline per image dimension, maybe it will be faster
 	switch (pc.perWG.imageDim)

include/nbl/builtin/hlsl/concepts/accessors/device_arithmetic.hlsl

@@ -0,0 +1,73 @@
+#ifndef _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_
+#define _NBL_BUILTIN_HLSL_CONCEPTS_ACCESSORS_DEVICE_ARITHMETIC_INCLUDED_
+
+#include "nbl/builtin/hlsl/concepts/accessors/generic_shared_data.hlsl"
+
+namespace nbl
+{
+namespace hlsl
+{
+namespace scan
+{
+
+#define NBL_CONCEPT_NAME GenericAtomicAccessor
+#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename)
+#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V)
+#define NBL_CONCEPT_PARAM_0 (accessor, T)
+#define NBL_CONCEPT_PARAM_1 (val, V)
+#define NBL_CONCEPT_PARAM_2 (index, uint64_t)
+NBL_CONCEPT_BEGIN(3)
+#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0
+#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1
+#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2
+NBL_CONCEPT_END(
+    ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(concepts::accessors::GenericDataAccessor, T, V, uint64_t))
+    ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicStore(index, val)), is_same_v, void))
+);
+#undef val
+#undef index
+#undef accessor
+#include <nbl/builtin/hlsl/concepts/__end.hlsl>
+
+#define NBL_CONCEPT_NAME DeviceReductionsAccessor
+#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename)
+#define NBL_CONCEPT_TPLT_PRM_NAMES (T)(V)
+#define NBL_CONCEPT_PARAM_0 (accessor, T)
+#define NBL_CONCEPT_PARAM_1 (val, V)
+#define NBL_CONCEPT_PARAM_2 (index, uint64_t)
+NBL_CONCEPT_BEGIN(3)
+#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0
+#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1
+#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2
+NBL_CONCEPT_END(
+    ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(GenericAtomicAccessor, T, V))
+    ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicMax(index, val)), is_same_v, V))
+);
+#undef val
+#undef index
+#undef accessor
+#include <nbl/builtin/hlsl/concepts/__end.hlsl>
+
+#define NBL_CONCEPT_NAME GenericAtomicArithmeticAccessor
+#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)
+#define NBL_CONCEPT_TPLT_PRM_NAMES (T)
+#define NBL_CONCEPT_PARAM_0 (accessor, T)
+#define NBL_CONCEPT_PARAM_1 (val, uint32_t)
+#define NBL_CONCEPT_PARAM_2 (index, uint64_t)
+NBL_CONCEPT_BEGIN(3)
+#define accessor NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0
+#define val NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1
+#define index NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2
+NBL_CONCEPT_END(
+    ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((accessor.atomicAdd(index, val)), is_same_v, uint32_t))
+);
+#undef val
+#undef index
+#undef accessor
+#include <nbl/builtin/hlsl/concepts/__end.hlsl>
+
+}
+}
+}
+
+#endif

include/nbl/builtin/hlsl/emulated/float64_t_impl.hlsl

@@ -4,7 +4,6 @@
 #include <nbl/builtin/hlsl/cpp_compat.hlsl>
 #include <nbl/builtin/hlsl/ieee754.hlsl>
 #include <nbl/builtin/hlsl/algorithm.hlsl>
-#include <nbl/builtin/hlsl/glsl_compat/core.hlsl>
 #include <nbl/builtin/hlsl/tgmath.hlsl>
 
 // TODO: when it will be possible, use this unions wherever they fit:

include/nbl/builtin/hlsl/functional.hlsl

@@ -5,7 +5,7 @@
 #define _NBL_BUILTIN_HLSL_FUNCTIONAL_INCLUDED_
 
 
-#include "nbl/builtin/hlsl/glsl_compat/core.hlsl"
+#include "nbl/builtin/hlsl/spirv_intrinsics/core.hlsl"
 #include "nbl/builtin/hlsl/limits.hlsl"
 #include "nbl/builtin/hlsl/concepts/vector.hlsl"
 #include "nbl/builtin/hlsl/array_accessors.hlsl"

include/nbl/builtin/hlsl/glsl_compat/core.hlsl

@@ -16,6 +16,11 @@ namespace nbl
 {
 namespace hlsl
 {
+// TODO: rename this namespace to `glsl_compat`
+// The functions in this namespace are meant for porting GLSL shaders with minimal changes.
+// Many of them are also handy utilities (default scopes and memory semantics, signed/unsigned/float dispatch, 16-bit emulation,
+// a C++ path) and get called all over the codebase because they are easier to use than the raw `spirv::` intrinsics.
+// Avoid calling them from code that isn't a GLSL port; those utilities need a home outside the `glsl` namespace.
 namespace glsl
 {
 
@@ -125,16 +130,6 @@ enable_if_t<spirv::is_pointer_v<Ptr_T>, T> atomicXor(Ptr_T ptr, T value)
 {
     return spirv::atomicXor<T, Ptr_T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
 }
-/* TODO: @Hazardu struct dispatchers like for `bitfieldExtract`
-template<typename T>
-T atomicMin(NBL_REF_ARG(T) ptr, T value)
-{
-}
-template<typename T>
-T atomicMax(NBL_REF_ARG(T) ptr, T value)
-{
-}
-*/
 // reference form: `ptr` is the variable itself, e.g. `buf[i]`, `bufferPtr.Get().member`, or a groupshared variable with `T` given explicitly
 template<typename T>
 T atomicExchange(NBL_REF_ARG(T) ptr, T value)
@@ -160,6 +155,100 @@ enable_if_t<spirv::is_pointer_v<Ptr_T>, T> atomicCompSwap(Ptr_T ptr, T comparato
     return spirv::atomicCompareExchange<T, Ptr_T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, spv::MemorySemanticsMaskNone, value, comparator);
 }
 
+namespace impl
+{
+template<typename T NBL_STRUCT_CONSTRAINABLE>
+struct atomicMin;
+
+template<typename T>
+NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral<T>)
+struct atomicMin<T NBL_PARTIAL_REQ_BOT(concepts::SignedIntegral<T>) >
+{
+    static T __call(NBL_REF_ARG(T) ptr, T value)
+    {
+        return spirv::atomicSMin<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+
+    template<typename Ptr_T> // DXC Workaround
+    static T __call(Ptr_T ptr, T value)
+    {
+        return spirv::atomicSMin<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+};
+
+template<typename T>
+NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral<T>)
+struct atomicMin<T NBL_PARTIAL_REQ_BOT(concepts::UnsignedIntegral<T>) >
+{
+    static T __call(NBL_REF_ARG(T) ptr, T value)
+    {
+        return spirv::atomicUMin<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+
+    template<typename Ptr_T> // DXC Workaround
+    static T __call(Ptr_T ptr, T value)
+    {
+        return spirv::atomicUMin<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+};
+
+template<typename T NBL_STRUCT_CONSTRAINABLE>
+struct atomicMax;
+
+template<typename T>
+NBL_PARTIAL_REQ_TOP(concepts::SignedIntegral<T>)
+struct atomicMax<T NBL_PARTIAL_REQ_BOT(concepts::SignedIntegral<T>) >
+{
+    static T __call(NBL_REF_ARG(T) ptr, T value)
+    {
+        return spirv::atomicSMax<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+
+    template<typename Ptr_T> // DXC Workaround
+    static T __call(Ptr_T ptr, T value)
+    {
+        return spirv::atomicSMax<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+};
+
+template<typename T>
+NBL_PARTIAL_REQ_TOP(concepts::UnsignedIntegral<T>)
+struct atomicMax<T NBL_PARTIAL_REQ_BOT(concepts::UnsignedIntegral<T>) >
+{
+    static T __call(NBL_REF_ARG(T) ptr, T value)
+    {
+        return spirv::atomicUMax<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+
+    template<typename Ptr_T> // DXC Workaround
+    static T __call(Ptr_T ptr, T value)
+    {
+        return spirv::atomicUMax<T>(ptr, spv::ScopeDevice, spv::MemorySemanticsMaskNone, value);
+    }
+};
+}
+
+template<typename T>
+T atomicMin(NBL_REF_ARG(T) ptr, T value)
+{
+    return impl::atomicMin<T>::__call(ptr, value);
+}
+template<typename T, typename Ptr_T> // DXC Workaround
+enable_if_t<spirv::is_pointer_v<Ptr_T>, T> atomicMin(Ptr_T ptr, T value)
+{
+    return impl::atomicMin<T>::template __call<Ptr_T>(ptr, value);
+}
+template<typename T>
+T atomicMax(NBL_REF_ARG(T) ptr, T value)
+{
+    return impl::atomicMax<T>::__call(ptr, value);
+}
+template<typename T, typename Ptr_T> // DXC Workaround
+enable_if_t<spirv::is_pointer_v<Ptr_T>, T> atomicMax(Ptr_T ptr, T value)
+{
+    return impl::atomicMax<T>::template __call<Ptr_T>(ptr, value);
+}
+
 /**
  * GLSL extended math
  */
@@ -189,7 +278,11 @@ uint32_t gl_InstanceIndex() {return spirv::InstanceIndex;}
 
 // TODO: Extemely annoying that HLSL doesn't have references, so we can't transparently alias the variables as `const&` :(
 uint32_t3 gl_NumWorkGroups() {return spirv::NumWorkGroups;}
-// TODO: DXC BUG prevents us from defining this!
+// Not a DXC bug: SPIR-V only lets a shader declare its workgroup size, there is nothing to read back.
+// `[numthreads]` emits the `LocalSize` execution mode, whose operands are literals that no instruction can reference.
+// `LocalSizeId` and the `WorkgroupSize` builtin take ids instead, but those must be constants (`WorkgroupSize` on an input
+// variable is invalid), so "reading" them just means reusing the constant that declared the size in the first place.
+// Hence the user defines this, returning the same compile-time constant they pass to `[numthreads]`.
 uint32_t3 gl_WorkGroupSize();
 uint32_t3 gl_WorkGroupID() {return spirv::WorkgroupId;}
 uint32_t3 gl_LocalInvocationID() {return spirv::LocalInvocationId;}
@@ -298,6 +391,8 @@ struct bitfieldExtract<T NBL_PARTIAL_REQ_BOT(concepts::UnsignedIntegral<typename
 };
 }
 
+// TODO: `bitfieldInsert` and `bitfieldExtract` are used as general utilities outside GLSL ports (16-bit emulation, C++ path),
+// they need a home outside the `glsl` namespace
 template<typename T>
 T bitfieldInsert(T base, T insert, uint32_t offset, uint32_t bits)
 {
@@ -413,6 +508,7 @@ struct notEqual_helper<Vectorial NBL_PARTIAL_REQ_BOT(concepts::Vectorial<Vectori
 #endif
 }
 
+// TODO: `equal` and `notEqual` are used as general utilities outside GLSL ports (C++ path), they need a home outside the `glsl` namespace
 template<typename T>
 inline vector<bool,vector_traits<T>::Dimension> equal(NBL_CONST_REF_ARG(T) x, NBL_CONST_REF_ARG(T) y)
 {

include/nbl/builtin/hlsl/glsl_compat/subgroup_arithmetic.hlsl

@@ -13,6 +13,8 @@ namespace hlsl
 namespace glsl
 {
 
+// TODO: these are used as general utilities outside GLSL ports (one name per operation for every type),
+// they need a home outside the `glsl` namespace, see the note above `namespace glsl` in glsl_compat/core.hlsl
 // TODO: @Hazardu this will need touching up in the impl when the `spirv::groupXXX` will change names to a predictable pattern from SPIR-V
 // TODO: Furthermore you'll need `bitfieldExtract`-like struct dispatcher to choose between int/float add/mul and sint/uint/float min/max
 template<typename T>

include/nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl

@@ -17,6 +17,7 @@ namespace glsl
 // TODO: Extemely annoying that HLSL doesn't have referencies, so we can't transparently alias the variables as `const&` :(
 // NOTE: These are not `uint16_t` even though they could be, because IIUtSC they're `uint32_t` in SPIR-V
 uint32_t gl_SubgroupSize() {return spirv::SubgroupSize;}
+// TODO: not a GLSL builtin, move it out of the `glsl` namespace to a better place
 uint32_t gl_SubgroupSizeLog2() {return firstbithigh(spirv::SubgroupSize);}
 uint32_t gl_SubgroupInvocationID() {return spirv::SubgroupLocalInvocationId;}
 

include/nbl/builtin/hlsl/memory_accessor.hlsl

@@ -4,7 +4,6 @@
 #ifndef _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_
 #define _NBL_BUILTIN_HLSL_MEMORY_ACCESSOR_INCLUDED_
 
-#include "nbl/builtin/hlsl/glsl_compat/core.hlsl"
 #include "nbl/builtin/hlsl/member_test_macros.hlsl"
 
 // weird namespace placing, see the comment where the macro is defined

include/nbl/builtin/hlsl/prefix_sum_blur/blur.hlsl

@@ -1,5 +1,4 @@
 #include "nbl/builtin/hlsl/cpp_compat.hlsl"
-#include "nbl/builtin/hlsl/glsl_compat/core.hlsl"
 #include "nbl/builtin/hlsl/workgroup/basic.hlsl"
 #include "nbl/builtin/hlsl/workgroup/arithmetic.hlsl"
 #include "nbl/builtin/hlsl/device_capabilities_traits.hlsl"