[hrx/gfx1151] DSV4.0 Flash optimizations

ggml/src/ggml-hrx/CMakeLists.txt

@@ -220,6 +220,8 @@ ggml_add_backend_library(ggml-hrx
     dispatch_registration/common/dispatch-get-rows.h
     dispatch_registration/common/dispatch-glu.cpp
     dispatch_registration/common/dispatch-glu.h
+    dispatch_registration/common/dispatch-hyper-connection.cpp
+    dispatch_registration/common/dispatch-hyper-connection.h
     dispatch_registration/common/dispatch-mul-mat-id.cpp
     dispatch_registration/common/dispatch-mul-mat-id-common.h
     dispatch_registration/common/dispatch-mul-mat-id.h
@@ -237,6 +239,8 @@ ggml_add_backend_library(ggml-hrx
     dispatch_registration/common/dispatch-scale.cpp
     dispatch_registration/common/dispatch-scale.h
     dispatch_registration/common/dispatch-unary.cpp
+    dispatch_registration/common/dispatch-reduction.cpp
+    dispatch_registration/common/dispatch-reduction.h
     dispatch_registration/common/dispatch-unary.h
     dispatch_registration/common/iq-codebook-resource.cpp
     dispatch_registration/common/iq-codebook-resource.h

ggml/src/ggml-hrx/README.md

@@ -1,5 +1,13 @@
 # HRX Backend
 
+## Activation precision
+
+For eligible shapes, Q8_0 matrix multiplication and MXFP4 expert matrix multiplication use Q8_1 activations. Small Q8_0 batches share weight loads across tokens; prefill uses matrix tiles. MXFP4 decode kernels can also share loads between tokens routed to the same expert. Dispatch uses tensor shapes, layouts, and precision requests, without model-specific settings.
+
+These routes require default accumulation precision and an activation precision request of undefined, Q8, or Q4. Explicit F32 accumulation or F32, F16, or BF16 activation requests retain the existing floating-point routes. Source precision is part of the imported graph parameters and graph-cache identity, so changing it requires a new dispatch plan.
+
+Small batched Q8_0 products can use F16 matrix accumulation with F32 reduction across contraction partitions under the same precision conditions. Explicit F32 accumulation retains the F32 path.
+
 ## Weight residency diagnostics
 
 HRX materializes imported host weights in the layout requested by the selected kernel and caches each result for the backend's lifetime. If one weight is materialized in two layouts, HRX warns about the duplicate allocations and their memory cost.
@@ -52,6 +60,8 @@ jq -c 'select(.event == "compile_request") | {cache_id, execution_id, kernel, in
 
 `hrx-minimum-commit.txt` records the minimum required commit in ROCm/hrx-system. It contains one full Git commit SHA. The backend's C++ APIs, Loom syntax, and numerical behavior can all depend on that revision. When `GGML_HRX=ON` and `HRX_SOURCE_DIR` is set, CMake requires the HRX checkout's HEAD to equal or descend from the recorded commit.
 
+The corpus uses typed `check.expect.approximate` operations introduced in [HRX d03575e84](https://github.com/ROCm/hrx-system/commit/d03575e84f602d3e39b87895a30fae214f308558).
+
 For builds using installed packages, check the source revision used to produce both HRX and Loom against this floor before validating a repair. From the staging checkout:
 
 ```sh

ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp

@@ -135,7 +135,8 @@ static bool can_elide_zero_output_node(const Graph &             graph,
 }
 
 static bool can_elide_node(const Graph & graph, const GraphNode & node, const std::vector<bool> & covered_nodes) {
-    return can_elide_layout_alias_node(graph, node, covered_nodes) ||
+    return (node.op == GGML_OP_NONE && node.inputs.empty()) ||
+           can_elide_layout_alias_node(graph, node, covered_nodes) ||
            can_elide_zero_output_node(graph, node, covered_nodes);
 }
 
@@ -204,10 +205,10 @@ bool DispatchScheduler::schedule_graph(Graph &                       graph,
                 covered_nodes[i] = true;
                 continue;
             }
-            plan_.status.append(pending_diagnostics);
-            plan_.status.append(match.status);
             const std::string message = unsupported_node_message(graph, i, *node);
             plan_.status.log("%s", message.c_str());
+            plan_.status.append(match.status);
+            plan_.status.append(pending_diagnostics);
             if (diagnostics != nullptr) {
                 diagnostics->unsupported_node_index = i;
                 diagnostics->unsupported_node       = node;

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-binary.cpp

@@ -193,18 +193,14 @@ static void add_binary_strided_parameters(Dispatch & dispatch,
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne0", std::to_string(output.ne[0]));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne1", std::to_string(output.ne[1]));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.ne2", std::to_string(output.ne[2]));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride1",
-                                               std::to_string(lhs.nb[1] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride2",
-                                               std::to_string(lhs.nb[2] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride3",
-                                               std::to_string(lhs.nb[3] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride1",
-                                               std::to_string(rhs.nb[1] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride2",
-                                               std::to_string(rhs.nb[2] / sizeof(float)));
-    dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride3",
-                                               std::to_string(rhs.nb[3] / sizeof(float)));
+    for (int dim = 1; dim < GGML_MAX_DIMS; ++dim) {
+        dispatch.kernel.compile_parameters.emplace(
+            "ggml.binary_f32.src0_stride" + std::to_string(dim),
+            std::to_string(broadcast_dim_flag(lhs, output, dim) ? 0 : lhs.nb[dim] / sizeof(float)));
+        dispatch.kernel.compile_parameters.emplace(
+            "ggml.binary_f32.src1_stride" + std::to_string(dim),
+            std::to_string(broadcast_dim_flag(rhs, output, dim) ? 0 : rhs.nb[dim] / sizeof(float)));
+    }
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_span",
                                                std::to_string(lhs_byte_count / sizeof(float)));
     dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_span",
@@ -298,9 +294,14 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     }
 
     const bool same_shape_inputs = same_shape(*lhs, *output) && same_shape(*rhs, *output);
+    const bool strided_broadcast = !same_shape_inputs && (!lhs->contiguous || !rhs->contiguous ||
+                                                          !packed_f32_layout(*lhs) || !packed_f32_layout(*rhs));
+    if (!binary_kind_allows_broadcast(params->op, *lhs, *rhs, *output)) {
+        return false;
+    }
     const CommonActivationPublicationCapabilities capabilities = {
-        binary_q8_publication_enabled(params->op, *output) ? DispatchActivationInputQ8_1X4 :
-                                                             DispatchActivationInputNone,
+        !strided_broadcast && binary_q8_publication_enabled(params->op, *output) ? DispatchActivationInputQ8_1X4 :
+                                                                                   DispatchActivationInputNone,
         DispatchActivationInputQ8_1X4,
     };
     const CommonActivationPublicationPlan publication_plan =
@@ -310,22 +311,21 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     const bool publish_q8 = q8_demand != nullptr;
 
     Dispatch dispatch;
-    if (same_shape_inputs) {
+    if (same_shape_inputs || strided_broadcast) {
         dispatch.kernel = make_kernel_specialization(publish_q8 ? kBinaryF32PublishQ8_1X4Kernel : kBinaryF32Kernel);
         add_binary_strided_parameters(dispatch, *lhs, *rhs, *output, lhs_byte_count, rhs_byte_count);
         if (publish_q8) {
             dispatch.kernel.integer_parameters.erase("element_count");
             dispatch.kernel.integer_parameters.emplace("token_count", output->ne[1]);
+        } else {
+            dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src0_stride0",
+                                                       std::to_string(broadcast_dim_flag(*lhs, *output, 0) ? 0 : 1));
+            dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.src1_stride0",
+                                                       std::to_string(broadcast_dim_flag(*rhs, *output, 0) ? 0 : 1));
         }
         dispatch.kernel.compile_parameters.emplace("ggml.binary_f32.op",
                                                    std::to_string(binary_kind_config_value(params->op)));
     } else {
-        if (!lhs->contiguous || !rhs->contiguous || !packed_f32_layout(*lhs) || !packed_f32_layout(*rhs)) {
-            return false;
-        }
-        if (!binary_kind_allows_broadcast(params->op, *lhs, *rhs, *output)) {
-            return false;
-        }
         dispatch.kernel = make_kernel_specialization(publish_q8 ? kBinaryBcF32PublishQ8_1X4Kernel : kBinaryBcF32Kernel);
         if (publish_q8) {
             dispatch.kernel.integer_parameters.emplace("token_count", output->ne[1]);
@@ -356,6 +356,33 @@ static bool match_binary_f32_dispatch(const DispatchMatchContext & context, Disp
     return match.status.success();
 }
 
+static bool match_add_f16(const DispatchMatchContext & context, DispatchMatch & match) {
+    const GraphNode * node = context.root_node;
+    if (node == nullptr || node->op != GGML_OP_ADD || node->inputs.size() != 2) {
+        return false;
+    }
+    const Value * lhs    = graph_value(context.graph, node->inputs[0]);
+    const Value * rhs    = graph_value(context.graph, node->inputs[1]);
+    const Value * output = graph_value(context.graph, node->output);
+    if (lhs == nullptr || rhs == nullptr || output == nullptr || lhs->type != GGML_TYPE_F16 ||
+        rhs->type != GGML_TYPE_F16 || output->type != GGML_TYPE_F16 || !lhs->contiguous || !rhs->contiguous ||
+        !output->contiguous || lhs->ne != output->ne || rhs->ne != output->ne || output->element_count < 1 ||
+        output->element_count > 134217728 || !binary_output_storage_is_safe(*lhs, *rhs, *output)) {
+        return false;
+    }
+    Dispatch dispatch;
+    dispatch.kernel = make_kernel_specialization(GGML_HRX_KERNEL_REF("loom_libs", "ggml_add_f16"));
+    dispatch.kernel.integer_parameters.emplace("element_count", output->element_count);
+    dispatch.bindings = {
+        { lhs->id,    0, lhs->byte_count    },
+        { rhs->id,    0, rhs->byte_count    },
+        { output->id, 0, output->byte_count }
+    };
+    match.dispatches.push_back(std::move(dispatch));
+    match.covered_nodes.push_back(context.root_index);
+    return true;
+}
+
 static void register_binary_dispatch_for(DispatchRegistryBuilder & registry, ggml_op root_op) {
     registry.add({
         "common.binary_f32",
@@ -370,6 +397,8 @@ static void register_binary_dispatch_for(DispatchRegistryBuilder & registry, ggm
 }  // namespace
 
 void register_binary_dispatch(DispatchRegistryBuilder & registry) {
+    registry.add(
+        { "common.add.f16", GGML_OP_ADD, DispatchMatchKind::SingleOp, 0, DispatchSource::Common, match_add_f16 });
     registry.add({
         "common.binary_swiglu_symmetric_i4_k32",
         GGML_OP_GLU,

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.cpp

@@ -8,16 +8,28 @@
 #include "dispatch-gather-add.h"
 #include "dispatch-get-rows.h"
 #include "dispatch-glu.h"
+#include "dispatch-hyper-connection.h"
 #include "dispatch-mul-mat-id.h"
 #include "dispatch-mul-mat.h"
+#include "dispatch-reduction.h"
 #include "dispatch-amax-scale.h"
 #include "dispatch-rmsnorm.h"
 #include "dispatch-rope-set-rows.h"
 #include "dispatch-scale.h"
 #include "dispatch-unary.h"
 
+#include <limits>
+#include <sstream>
+
 namespace ggml::hrx {
 
+std::string float_config_value(float value) {
+    std::ostringstream out;
+    out.precision(std::numeric_limits<float>::max_digits10);
+    out << value;
+    return out.str();
+}
+
 void register_common_dispatches(DispatchRegistryBuilder & registry) {
     register_binary_dispatch(registry);
     register_copy_dispatch(registry);
@@ -27,8 +39,10 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) {
     register_gather_add_dispatch(registry);
     register_get_rows_dispatches(registry);
     register_glu_dispatches(registry);
+    register_hyper_connection_dispatches(registry);
     register_mul_mat_id_dispatches(registry);
     register_mul_mat_dispatches(registry);
+    register_reduction_dispatches(registry);
     register_amax_scale_dispatch(registry);
     register_rope_set_rows_dispatches(registry);
     register_scale_dispatch(registry);

ggml/src/ggml-hrx/dispatch_registration/common/dispatch-common.h

@@ -2,8 +2,12 @@
 
 #include "../dispatch-registry.h"
 
+#include <string>
+
 namespace ggml::hrx {
 
+std::string float_config_value(float value);
+
 void register_common_dispatches(DispatchRegistryBuilder & registry);
 
 }  // namespace ggml::hrx