Skip to content

Commit 637c056

Browse files
lgirdwoodlgirdwood
committed
audio: modules: optimize loop unrolling and guards for RISC-V dual-issue pipelines
Optimize generic processing paths for dual-issue RISC-V execution pipelines: - asrc: 4-way loop unrolling in asrc_fir_filter16() and asrc_fir_filter32(). - level_multiplier: add SOF_USE_RISCV_SIMD(VOLUME) guards and 4-way loop unrolling for s16, s24, and s32 integer gain scaling. - pcm_converter: 4x and 8x loop unrolling for integer conversions (s16_to_s24, s24_to_s16, s16_to_s32, s32_to_s16, s24_to_s32, s32_to_s24). - eq_fir: update fallback guards to include SOF_USE_RISCV_SIMD(FILTER). Signed-off-by: Liam Girdwood <lgirdwood@gmail.com>
1 parent 3aedae6 commit 637c056

5 files changed

Lines changed: 140 additions & 53 deletions

File tree

‎src/audio/asrc/asrc_farrow_generic.c‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,16 @@ void asrc_fir_filter16(struct asrc_farrow *src_obj, int16_t **output_buffers,
4040
/* Initialise the accumulator */
4141
prod = 0;
4242

43-
/* Iterate over the filter bins.
44-
* Data is Q1.15, coefficients are Q1.30. Prod will be Qx.45.
45-
*/
46-
for (n = 0; n < src_obj->filter_length; n++)
43+
/* 4-way unrolled convolution for ILP / RISC-V SIMD pipeline */
44+
for (n = 0; n <= src_obj->filter_length - 4; n += 4) {
45+
prod += (int64_t)buffer_p[0] * filter_p[0];
46+
prod += (int64_t)buffer_p[1] * filter_p[1];
47+
prod += (int64_t)buffer_p[2] * filter_p[2];
48+
prod += (int64_t)buffer_p[3] * filter_p[3];
49+
buffer_p += 4;
50+
filter_p += 4;
51+
}
52+
for (; n < src_obj->filter_length; n++)
4753
prod += (int64_t)(*buffer_p++) * (*filter_p++);
4854

4955
/* Shift left after accumulation, because interim
@@ -88,14 +94,16 @@ void asrc_fir_filter32(struct asrc_farrow *src_obj, int32_t **output_buffers,
8894
/* Initialise the accumulator */
8995
prod = 0;
9096

91-
/* Iterate over the filter bins. Data is Q1.31, coefficients
92-
* are Q1.22. They are down scaled by 1 shift. In addition
93-
* there C is implementation specific right shift by 8. It
94-
* gives headroom to calculate up to 256 taps FIR. The use
95-
* of 24 bits of 32 bits is not a practical limitation for
96-
* quality. The product is Qx.54.
97-
*/
98-
for (n = 0; n < src_obj->filter_length; n++)
97+
/* 4-way unrolled convolution for ILP / RISC-V SIMD pipeline */
98+
for (n = 0; n <= src_obj->filter_length - 4; n += 4) {
99+
prod += (int64_t)buffer_p[0] * (filter_p[0] >> 8);
100+
prod += (int64_t)buffer_p[1] * (filter_p[1] >> 8);
101+
prod += (int64_t)buffer_p[2] * (filter_p[2] >> 8);
102+
prod += (int64_t)buffer_p[3] * (filter_p[3] >> 8);
103+
buffer_p += 4;
104+
filter_p += 4;
105+
}
106+
for (; n < src_obj->filter_length; n++)
99107
prod += (int64_t)(*buffer_p++) * (*filter_p++ >> 8);
100108

101109
/* Shift left after accumulation, because interim

‎src/audio/eq_fir/eq_fir.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
#include <sof/math/fir_config.h>
1515
#include <sof/common.h>
1616

17-
#if SOF_USE_HIFI(NONE, FILTER) || SOF_USE_ARM_SIMD(FILTER)
17+
#if SOF_USE_HIFI(NONE, FILTER) || SOF_USE_ARM_SIMD(FILTER) || SOF_USE_RISCV_SIMD(FILTER)
1818
#include <sof/math/fir_generic.h>
1919
#endif
2020
#if SOF_USE_HIFI(2, FILTER)

‎src/audio/eq_fir/eq_fir_generic.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include <sof/math/fir_config.h>
1010
#include <sof/common.h>
1111

12-
#if SOF_USE_HIFI(NONE, FILTER) || SOF_USE_ARM_SIMD(FILTER)
12+
#if SOF_USE_HIFI(NONE, FILTER) || SOF_USE_ARM_SIMD(FILTER) || SOF_USE_RISCV_SIMD(FILTER)
1313

1414
#include <sof/audio/module_adapter/module/generic.h>
1515
#include <sof/math/fir_generic.h>
@@ -139,4 +139,4 @@ void eq_fir_s32(struct fir_state_32x16 fir[], const struct cir_buf_source *sourc
139139
}
140140
#endif /* CONFIG_FORMAT_S32LE */
141141

142-
#endif /* FILTER_HIFI_NONE */
142+
#endif /* SOF_USE_HIFI(NONE, FILTER) || SOF_USE_ARM_SIMD(FILTER) || SOF_USE_RISCV_SIMD(FILTER) */

‎src/audio/level_multiplier/level_multiplier-generic.c‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
#define LEVEL_MULTIPLIER_S24_SHIFT Q_SHIFT_BITS_64(23, LEVEL_MULTIPLIER_QXY_Y, 23)
1515
#define LEVEL_MULTIPLIER_S32_SHIFT Q_SHIFT_BITS_64(31, LEVEL_MULTIPLIER_QXY_Y, 31)
1616

17-
#if SOF_USE_HIFI(NONE, VOLUME) && !SOF_USE_ARM_SIMD(VOLUME)
17+
#if (SOF_USE_HIFI(NONE, VOLUME) || SOF_USE_RISCV_SIMD(VOLUME)) && !SOF_USE_ARM_SIMD(VOLUME)
1818

1919
#if CONFIG_FORMAT_S16LE
2020
/**
@@ -66,10 +66,17 @@ static int level_multiplier_s16(const struct processing_module *mod,
6666
samples_without_wrap = y_end - y;
6767
samples_without_wrap = MIN(samples_without_wrap, source_samples_without_wrap);
6868
samples_without_wrap = MIN(samples_without_wrap, remaining_samples);
69-
for (i = 0; i < samples_without_wrap; i++) {
70-
*y = q_multsr_sat_32x32_16(*x, gain, LEVEL_MULTIPLIER_S16_SHIFT);
71-
x++;
72-
y++;
69+
int samples4 = samples_without_wrap & ~3;
70+
for (i = 0; i < samples4; i += 4) {
71+
y[0] = q_multsr_sat_32x32_16(x[0], gain, LEVEL_MULTIPLIER_S16_SHIFT);
72+
y[1] = q_multsr_sat_32x32_16(x[1], gain, LEVEL_MULTIPLIER_S16_SHIFT);
73+
y[2] = q_multsr_sat_32x32_16(x[2], gain, LEVEL_MULTIPLIER_S16_SHIFT);
74+
y[3] = q_multsr_sat_32x32_16(x[3], gain, LEVEL_MULTIPLIER_S16_SHIFT);
75+
x += 4;
76+
y += 4;
77+
}
78+
for (; i < samples_without_wrap; i++) {
79+
*y++ = q_multsr_sat_32x32_16(*x++, gain, LEVEL_MULTIPLIER_S16_SHIFT);
7380
}
7481

7582
/* One of the buffers needs a wrap (or end of data), so check for wrap */
@@ -136,11 +143,22 @@ static int level_multiplier_s24(const struct processing_module *mod,
136143
samples_without_wrap = y_end - y;
137144
samples_without_wrap = MIN(samples_without_wrap, source_samples_without_wrap);
138145
samples_without_wrap = MIN(samples_without_wrap, remaining_samples);
139-
for (i = 0; i < samples_without_wrap; i++) {
140-
*y = q_multsr_sat_32x32_24(sign_extend_s24(*x), gain,
146+
int samples4 = samples_without_wrap & ~3;
147+
for (i = 0; i < samples4; i += 4) {
148+
y[0] = q_multsr_sat_32x32_24(sign_extend_s24(x[0]), gain,
149+
LEVEL_MULTIPLIER_S24_SHIFT);
150+
y[1] = q_multsr_sat_32x32_24(sign_extend_s24(x[1]), gain,
151+
LEVEL_MULTIPLIER_S24_SHIFT);
152+
y[2] = q_multsr_sat_32x32_24(sign_extend_s24(x[2]), gain,
153+
LEVEL_MULTIPLIER_S24_SHIFT);
154+
y[3] = q_multsr_sat_32x32_24(sign_extend_s24(x[3]), gain,
155+
LEVEL_MULTIPLIER_S24_SHIFT);
156+
x += 4;
157+
y += 4;
158+
}
159+
for (; i < samples_without_wrap; i++) {
160+
*y++ = q_multsr_sat_32x32_24(sign_extend_s24(*x++), gain,
141161
LEVEL_MULTIPLIER_S24_SHIFT);
142-
x++;
143-
y++;
144162
}
145163

146164
/* One of the buffers needs a wrap (or end of data), so check for wrap */
@@ -207,10 +225,17 @@ static int level_multiplier_s32(const struct processing_module *mod,
207225
samples_without_wrap = y_end - y;
208226
samples_without_wrap = MIN(samples_without_wrap, source_samples_without_wrap);
209227
samples_without_wrap = MIN(samples_without_wrap, remaining_samples);
210-
for (i = 0; i < samples_without_wrap; i++) {
211-
*y = q_multsr_sat_32x32(*x, gain, LEVEL_MULTIPLIER_S32_SHIFT);
212-
x++;
213-
y++;
228+
int samples4 = samples_without_wrap & ~3;
229+
for (i = 0; i < samples4; i += 4) {
230+
y[0] = q_multsr_sat_32x32(x[0], gain, LEVEL_MULTIPLIER_S32_SHIFT);
231+
y[1] = q_multsr_sat_32x32(x[1], gain, LEVEL_MULTIPLIER_S32_SHIFT);
232+
y[2] = q_multsr_sat_32x32(x[2], gain, LEVEL_MULTIPLIER_S32_SHIFT);
233+
y[3] = q_multsr_sat_32x32(x[3], gain, LEVEL_MULTIPLIER_S32_SHIFT);
234+
x += 4;
235+
y += 4;
236+
}
237+
for (; i < samples_without_wrap; i++) {
238+
*y++ = q_multsr_sat_32x32(*x++, gain, LEVEL_MULTIPLIER_S32_SHIFT);
214239
}
215240

216241
/* One of the buffers needs a wrap (or end of data), so check for wrap */
@@ -263,4 +288,4 @@ level_multiplier_func level_multiplier_find_proc_func(enum sof_ipc_frame src_fmt
263288
return NULL;
264289
}
265290

266-
#endif /* SOF_USE_HIFI(NONE, VOLUME) */
291+
#endif /* (SOF_USE_HIFI(NONE, VOLUME) || SOF_USE_RISCV_SIMD(VOLUME)) && !SOF_USE_ARM_SIMD(VOLUME) */

‎src/audio/pcm_converter/pcm_converter_generic.c‎

Lines changed: 78 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,21 @@ static int pcm_convert_s16_to_s24(const struct cir_buf_source *source,
222222
n = MIN(n, nmax);
223223
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S32_SAMPLES;
224224
n = MIN(n, nmax);
225-
for (i = 0; i < n; i++) {
226-
*dst = *src << 8;
227-
src++;
228-
dst++;
225+
size_t n8 = n & ~7;
226+
for (i = 0; i < n8; i += 8) {
227+
dst[0] = (int32_t)src[0] << 8;
228+
dst[1] = (int32_t)src[1] << 8;
229+
dst[2] = (int32_t)src[2] << 8;
230+
dst[3] = (int32_t)src[3] << 8;
231+
dst[4] = (int32_t)src[4] << 8;
232+
dst[5] = (int32_t)src[5] << 8;
233+
dst[6] = (int32_t)src[6] << 8;
234+
dst[7] = (int32_t)src[7] << 8;
235+
src += 8;
236+
dst += 8;
237+
}
238+
for (; i < n; i++) {
239+
*dst++ = (int32_t)*src++ << 8;
229240
}
230241
}
231242

@@ -249,10 +260,17 @@ static int pcm_convert_s24_to_s16(const struct cir_buf_source *source,
249260
n = MIN(n, nmax);
250261
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S16_SAMPLES;
251262
n = MIN(n, nmax);
252-
for (i = 0; i < n; i++) {
253-
*dst = sat_int16(Q_SHIFT_RND(sign_extend_s24(*src), 23, 15));
254-
src++;
255-
dst++;
263+
size_t n4 = n & ~3;
264+
for (i = 0; i < n4; i += 4) {
265+
dst[0] = sat_int16(Q_SHIFT_RND(sign_extend_s24(src[0]), 23, 15));
266+
dst[1] = sat_int16(Q_SHIFT_RND(sign_extend_s24(src[1]), 23, 15));
267+
dst[2] = sat_int16(Q_SHIFT_RND(sign_extend_s24(src[2]), 23, 15));
268+
dst[3] = sat_int16(Q_SHIFT_RND(sign_extend_s24(src[3]), 23, 15));
269+
src += 4;
270+
dst += 4;
271+
}
272+
for (; i < n; i++) {
273+
*dst++ = sat_int16(Q_SHIFT_RND(sign_extend_s24(*src++), 23, 15));
256274
}
257275
}
258276

@@ -280,10 +298,21 @@ static int pcm_convert_s16_to_s32(const struct cir_buf_source *source,
280298
n = MIN(n, nmax);
281299
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S32_SAMPLES;
282300
n = MIN(n, nmax);
283-
for (i = 0; i < n; i++) {
284-
*dst = *src << 16;
285-
src++;
286-
dst++;
301+
size_t n8 = n & ~7;
302+
for (i = 0; i < n8; i += 8) {
303+
dst[0] = (int32_t)src[0] << 16;
304+
dst[1] = (int32_t)src[1] << 16;
305+
dst[2] = (int32_t)src[2] << 16;
306+
dst[3] = (int32_t)src[3] << 16;
307+
dst[4] = (int32_t)src[4] << 16;
308+
dst[5] = (int32_t)src[5] << 16;
309+
dst[6] = (int32_t)src[6] << 16;
310+
dst[7] = (int32_t)src[7] << 16;
311+
src += 8;
312+
dst += 8;
313+
}
314+
for (; i < n; i++) {
315+
*dst++ = (int32_t)*src++ << 16;
287316
}
288317
}
289318

@@ -307,10 +336,17 @@ static int pcm_convert_s32_to_s16(const struct cir_buf_source *source,
307336
n = MIN(n, nmax);
308337
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S16_SAMPLES;
309338
n = MIN(n, nmax);
310-
for (i = 0; i < n; i++) {
311-
*dst = sat_int16(Q_SHIFT_RND(*src, 31, 15));
312-
src++;
313-
dst++;
339+
size_t n4 = n & ~3;
340+
for (i = 0; i < n4; i += 4) {
341+
dst[0] = sat_int16(Q_SHIFT_RND(src[0], 31, 15));
342+
dst[1] = sat_int16(Q_SHIFT_RND(src[1], 31, 15));
343+
dst[2] = sat_int16(Q_SHIFT_RND(src[2], 31, 15));
344+
dst[3] = sat_int16(Q_SHIFT_RND(src[3], 31, 15));
345+
src += 4;
346+
dst += 4;
347+
}
348+
for (; i < n; i++) {
349+
*dst++ = sat_int16(Q_SHIFT_RND(*src++, 31, 15));
314350
}
315351
}
316352

@@ -338,10 +374,21 @@ static int pcm_convert_s24_to_s32(const struct cir_buf_source *source,
338374
n = MIN(n, nmax);
339375
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S32_SAMPLES;
340376
n = MIN(n, nmax);
341-
for (i = 0; i < n; i++) {
342-
*dst = *src << 8;
343-
src++;
344-
dst++;
377+
size_t n8 = n & ~7;
378+
for (i = 0; i < n8; i += 8) {
379+
dst[0] = (int32_t)src[0] << 8;
380+
dst[1] = (int32_t)src[1] << 8;
381+
dst[2] = (int32_t)src[2] << 8;
382+
dst[3] = (int32_t)src[3] << 8;
383+
dst[4] = (int32_t)src[4] << 8;
384+
dst[5] = (int32_t)src[5] << 8;
385+
dst[6] = (int32_t)src[6] << 8;
386+
dst[7] = (int32_t)src[7] << 8;
387+
src += 8;
388+
dst += 8;
389+
}
390+
for (; i < n; i++) {
391+
*dst++ = (int32_t)*src++ << 8;
345392
}
346393
}
347394

@@ -365,10 +412,17 @@ static int pcm_convert_s32_to_s24(const struct cir_buf_source *source,
365412
n = MIN(n, nmax);
366413
nmax = cir_buf_bytes_without_wrap(dst, sink->buf_end) >> BYTES_TO_S32_SAMPLES;
367414
n = MIN(n, nmax);
368-
for (i = 0; i < n; i++) {
369-
*dst = sat_int24(Q_SHIFT_RND(*src, 31, 23));
370-
src++;
371-
dst++;
415+
size_t n4 = n & ~3;
416+
for (i = 0; i < n4; i += 4) {
417+
dst[0] = sat_int24(Q_SHIFT_RND(src[0], 31, 23));
418+
dst[1] = sat_int24(Q_SHIFT_RND(src[1], 31, 23));
419+
dst[2] = sat_int24(Q_SHIFT_RND(src[2], 31, 23));
420+
dst[3] = sat_int24(Q_SHIFT_RND(src[3], 31, 23));
421+
src += 4;
422+
dst += 4;
423+
}
424+
for (; i < n; i++) {
425+
*dst++ = sat_int24(Q_SHIFT_RND(*src++, 31, 23));
372426
}
373427
}
374428

0 commit comments

Comments
 (0)