From c6e21f071ab9b292d79cf971d31b1d728bf31e89 Mon Sep 17 00:00:00 2001 From: Yicheng Kang <3193997905@qq.com> Date: Sun, 25 Oct 2026 10:34:12 -0700 Subject: [PATCH] RISC-V: return the first occurrence on ties in the RVV index kernels The i?amax/i?amin/i?max/i?min (and complex icamax/izamax/icamin/izamin) RVV kernels track, per vector lane, the index of the first extreme value that lane has seen, and then return the index held by the lowest lane containing the overall extreme (vfirst + vslidedown). When the extreme value occurs more than once, the lowest such lane can hold a later index than another lane, so the kernels did not return the first occurrence as BLAS requires (for example isamax returned 64 instead of 21 for n = 65 with equal maxima at 21 and 64). Select the smallest recorded index among the lanes holding the extreme value instead (vmerge with UINT_MAX + vredminu). --- kernel/riscv64/iamax_rvv.c | 21 +++++++++++++++++---- kernel/riscv64/iamin_rvv.c | 21 +++++++++++++++++---- kernel/riscv64/imax_rvv.c | 21 +++++++++++++++++---- kernel/riscv64/imin_rvv.c | 21 +++++++++++++++++---- kernel/riscv64/izamax_rvv.c | 21 +++++++++++++++++---- kernel/riscv64/izamin_rvv.c | 21 +++++++++++++++++---- 6 files changed, 102 insertions(+), 24 deletions(-) diff --git a/kernel/riscv64/iamax_rvv.c b/kernel/riscv64/iamax_rvv.c index 8362d7cefd..4a1d1ff503 100644 --- a/kernel/riscv64/iamax_rvv.c +++ b/kernel/riscv64/iamax_rvv.c @@ -53,6 +53,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m8 #define VMVVXS_UINT __riscv_vmv_x_s_u64m8_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m8_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m8(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m8() @@ -79,6 +85,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m8 #define VMVVXS_UINT __riscv_vmv_x_s_u32m8_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m8_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -140,10 +152,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMAXVS_FLOAT(v_max, v_res, vlmax); maxf = VFMVFS_FLOAT_M1(v_res); mask = VMFGEVF_FLOAT(v_max, maxf, vlmax); - max_index = VFIRSTM(mask, vlmax); - - v_max_index = VSLIDEDOWN_UINT(v_max_index, max_index, vlmax); - max_index = VMVVXS_UINT(v_max_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_max_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_max_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_max_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + max_index = VMVXS_UINT_M1(v_first); return(max_index+1); } diff --git a/kernel/riscv64/iamin_rvv.c b/kernel/riscv64/iamin_rvv.c index f90dbb5450..9fcb1e8266 100644 --- a/kernel/riscv64/iamin_rvv.c +++ b/kernel/riscv64/iamin_rvv.c @@ -54,6 +54,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m8 #define VMVVXS_UINT __riscv_vmv_x_s_u64m8_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m8_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m8(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m8() @@ -80,6 +86,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m8 #define VMVVXS_UINT __riscv_vmv_x_s_u32m8_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m8_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -141,10 +153,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMINVS_FLOAT(v_min, v_res, vlmax); minf = VFMVFS_FLOAT_M1(v_res); mask = VMFLEVF_FLOAT(v_min, minf, vlmax); - min_index = VFIRSTM(mask, vlmax); - - v_min_index = VSLIDEDOWN_UINT(v_min_index, min_index, vlmax); - min_index = VMVVXS_UINT(v_min_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_min_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_min_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_min_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + min_index = VMVXS_UINT_M1(v_first); return(min_index+1); } diff --git a/kernel/riscv64/imax_rvv.c b/kernel/riscv64/imax_rvv.c index b1a77b1783..206922ba87 100644 --- a/kernel/riscv64/imax_rvv.c +++ b/kernel/riscv64/imax_rvv.c @@ -53,6 +53,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m8 #define VMVVXS_UINT __riscv_vmv_x_s_u64m8_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m8_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m8(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m8() @@ -78,6 +84,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m8 #define VMVVXS_UINT __riscv_vmv_x_s_u32m8_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m8_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -137,10 +149,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMAXVS_FLOAT(v_max, v_res, vlmax); maxf = VFMVFS_FLOAT_M1(v_res); mask = VMFGEVF_FLOAT(v_max, maxf, vlmax); - max_index = VFIRSTM(mask, vlmax); - - v_max_index = VSLIDEDOWN_UINT(v_max_index, max_index, vlmax); - max_index = VMVVXS_UINT(v_max_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_max_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_max_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_max_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + max_index = VMVXS_UINT_M1(v_first); return(max_index+1); } diff --git a/kernel/riscv64/imin_rvv.c b/kernel/riscv64/imin_rvv.c index 1de7f3233b..2a638b0841 100644 --- a/kernel/riscv64/imin_rvv.c +++ b/kernel/riscv64/imin_rvv.c @@ -53,6 +53,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m8 #define VMVVXS_UINT __riscv_vmv_x_s_u64m8_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m8_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m8(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m8() @@ -78,6 +84,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m8 #define VMVVXS_UINT __riscv_vmv_x_s_u32m8_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m8 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m8_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -137,10 +149,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMINVS_FLOAT(v_min, v_res, vlmax); minf = VFMVFS_FLOAT_M1(v_res); mask = VMFLEVF_FLOAT(v_min, minf, vlmax); - min_index = VFIRSTM(mask, vlmax); - - v_min_index = VSLIDEDOWN_UINT(v_min_index, min_index, vlmax); - min_index = VMVVXS_UINT(v_min_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_min_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_min_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_min_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + min_index = VMVXS_UINT_M1(v_first); return(min_index+1); } diff --git a/kernel/riscv64/izamax_rvv.c b/kernel/riscv64/izamax_rvv.c index 32f66a7a7a..27bead5425 100644 --- a/kernel/riscv64/izamax_rvv.c +++ b/kernel/riscv64/izamax_rvv.c @@ -58,6 +58,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m4 #define VMVVXS_UINT __riscv_vmv_x_s_u64m4_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m4 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m4_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m4(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m4() @@ -89,6 +95,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m4 #define VMVVXS_UINT __riscv_vmv_x_s_u32m4_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m4 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m4_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -163,10 +175,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMAXVS_FLOAT(v_max, v_res, vlmax); maxf = VFMVFS_FLOAT_M1(v_res); mask = VMFGEVF_FLOAT(v_max, maxf, vlmax); - max_index = VFIRSTM(mask, vlmax); - - v_max_index = VSLIDEDOWN_UINT(v_max_index, max_index, vlmax); - max_index = VMVVXS_UINT(v_max_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_max_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_max_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_max_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + max_index = VMVXS_UINT_M1(v_first); return(max_index+1); } diff --git a/kernel/riscv64/izamin_rvv.c b/kernel/riscv64/izamin_rvv.c index d34b220fa6..0f4e42654f 100644 --- a/kernel/riscv64/izamin_rvv.c +++ b/kernel/riscv64/izamin_rvv.c @@ -57,6 +57,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f64m1_f64 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u64m4 #define VMVVXS_UINT __riscv_vmv_x_s_u64m4_u64 +#define VMERGE_UINT __riscv_vmerge_vvm_u64m4 +#define VREDMINU_UINT __riscv_vredminu_vs_u64m4_u64m1 +#define UINT_V_T_M1 vuint64m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u64m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u64m1_u64 +#define INDEX_MAX 0xffffffffffffffffULL #else #define VSETVL(n) __riscv_vsetvl_e32m4(n) #define VSETVL_MAX __riscv_vsetvlmax_e32m4() @@ -86,6 +92,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define VFMVFS_FLOAT_M1 __riscv_vfmv_f_s_f32m1_f32 #define VSLIDEDOWN_UINT __riscv_vslidedown_vx_u32m4 #define VMVVXS_UINT __riscv_vmv_x_s_u32m4_u32 +#define VMERGE_UINT __riscv_vmerge_vvm_u32m4 +#define VREDMINU_UINT __riscv_vredminu_vs_u32m4_u32m1 +#define UINT_V_T_M1 vuint32m1_t +#define VMVVX_UINT_M1 __riscv_vmv_v_x_u32m1 +#define VMVXS_UINT_M1 __riscv_vmv_x_s_u32m1_u32 +#define INDEX_MAX 0xffffffffU #endif BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) @@ -162,10 +174,11 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) v_res = VFREDMINVS_FLOAT(v_min, v_res, vlmax); minf = VFMVFS_FLOAT_M1(v_res); mask = VMFLEVF_FLOAT(v_min, minf, vlmax); - min_index = VFIRSTM(mask, vlmax); - - v_min_index = VSLIDEDOWN_UINT(v_min_index, min_index, vlmax); - min_index = VMVVXS_UINT(v_min_index); + // Several lanes can hold the extreme value; BLAS returns its first occurrence, which is the + // smallest index recorded by those lanes (each lane keeps the first occurrence it has seen). + v_min_index = VMERGE_UINT(VMVVX_UINT(INDEX_MAX, vlmax), v_min_index, mask, vlmax); + UINT_V_T_M1 v_first = VREDMINU_UINT(v_min_index, VMVVX_UINT_M1(INDEX_MAX, 1), vlmax); + min_index = VMVXS_UINT_M1(v_first); return(min_index+1); }