phynexis-deps/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix RVV TRSM packed-A traversal

Browse Source 
Tile packed-A traversal by GEMM_UNROLL_M so the RVV TRSM solver matches the GEMM/itcopy packing contract on non-x280 RISC-V targets. This keeps the RVV solve path while avoiding VLEN-dependent corruption when VSETVL_MAX differs from the target GEMM unroll width.

Signed-off-by: Xiaofei Gong <gongxiaofei24@iscas.ac.cn>
Signed-off-by: YuanSheng <yuansheng@isrc.iscas.ac.cn>
This commit is contained in:
Felix-Gong
2026-07-19 14:43:14 +00:00
parent a7d62fe5de
commit 837c5f67d3
4 changed files with 383 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/riscv64/trsm_kernel_LN_rvv_v1.c
+96 -75
View File
@@ -71,6 +71,30 @@ static FLOAT dm1 = -1.;
#define GEMM_KERNEL GEMM_KERNEL_N
#endif
#if GEMM_DEFAULT_UNROLL_M == 1
#define GEMM_UNROLL_M_SHIFT 0
#endif
#if GEMM_DEFAULT_UNROLL_M == 2
#define GEMM_UNROLL_M_SHIFT 1
#endif
#if GEMM_DEFAULT_UNROLL_M == 4
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 6
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 8
#define GEMM_UNROLL_M_SHIFT 3
#endif
#if GEMM_DEFAULT_UNROLL_M == 16
#define GEMM_UNROLL_M_SHIFT 4
#endif
#if GEMM_DEFAULT_UNROLL_N == 1
#define GEMM_UNROLL_N_SHIFT 0
#endif
@@ -91,7 +115,9 @@ static FLOAT dm1 = -1.;
#define GEMM_UNROLL_N_SHIFT 4
#endif
// Optimizes the implementation in ../arm64/trsm_kernel_LN_sve.c
// Packed A is tiled by GEMM_UNROLL_M (the GEMM micro-kernel / itcopy contract),
// NOT by runtime VL: tiling by VSETVL_MAX only matches x280 and silently
// corrupts on other targets (e.g. zvl128b, unroll 8). Only solve() vectorizes.
#ifndef COMPLEX
@@ -217,10 +243,6 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
BLASLONG i, j;
FLOAT *aa, *cc;
BLASLONG kk;
size_t vl = VSETVL_MAX;
//fprintf(stderr, "%s , %s, m = %4ld n = %4ld k = %4ld offset = %4ld\n", __FILE__, __FUNCTION__, m, n, k, offset); // Debug
j = (n >> GEMM_UNROLL_N_SHIFT);
@@ -228,63 +250,62 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
kk = m + offset;
i = m % vl;
if (i) {
aa = a + (m - i) * k * COMPSIZE;
cc = c + (m - i) * COMPSIZE;
if (m & (GEMM_UNROLL_M - 1)) {
for (i = 1; i < GEMM_UNROLL_M; i *= 2){
if (m & i) {
aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;
cc = c + ((m & ~(i - 1)) - i) * COMPSIZE;
if (k - kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,
if (k - kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa + i * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
aa + i * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
solve(i, GEMM_UNROLL_N,
aa + (kk - i) * i * COMPSIZE,
b + (kk - i) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
kk -= i;
}
}
solve(i, GEMM_UNROLL_N,
aa + (kk - i) * i * COMPSIZE,
b + (kk - i) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
kk -= i;
}
int mod = i;
i = vl;
if (i <= m) {
aa = a + (m - mod - vl) * k * COMPSIZE;
cc = c + (m - mod - vl) * COMPSIZE;
i = (m >> GEMM_UNROLL_M_SHIFT);
if (i > 0) {
aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE;
cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * COMPSIZE;
do {
if (k - kk > 0) {
GEMM_KERNEL(vl, GEMM_UNROLL_N, k - kk, dm1,
GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1,
#ifdef COMPLEX
ZERO,
#endif
aa + vl * kk * COMPSIZE,
aa + GEMM_UNROLL_M * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
solve(vl, GEMM_UNROLL_N,
aa + (kk - vl) * vl * COMPSIZE,
b + (kk - vl) * GEMM_UNROLL_N * COMPSIZE,
solve(GEMM_UNROLL_M, GEMM_UNROLL_N,
aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE,
b + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa -= vl * k * COMPSIZE;
cc -= vl * COMPSIZE;
kk -= vl;
i += vl;
} while (i <= m);
aa -= GEMM_UNROLL_M * k * COMPSIZE;
cc -= GEMM_UNROLL_M * COMPSIZE;
kk -= GEMM_UNROLL_M;
i --;
} while (i > 0);
}
b += GEMM_UNROLL_N * k * COMPSIZE;
c += GEMM_UNROLL_N * ldc * COMPSIZE;
j --;
@@ -298,59 +319,59 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
kk = m + offset;
i = m % vl;
if (i) {
aa = a + (m - i) * k * COMPSIZE;
cc = c + (m - i) * COMPSIZE;
if (m & (GEMM_UNROLL_M - 1)) {
for (i = 1; i < GEMM_UNROLL_M; i *= 2){
if (m & i) {
aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;
cc = c + ((m & ~(i - 1)) - i) * COMPSIZE;
if (k - kk > 0) {
GEMM_KERNEL(i, j, k - kk, dm1,
if (k - kk > 0) {
GEMM_KERNEL(i, j, k - kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa + i * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc, ldc);
aa + i * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc, ldc);
}
solve(i, j,
aa + (kk - i) * i * COMPSIZE,
b + (kk - i) * j * COMPSIZE,
cc, ldc);
kk -= i;
}
}
solve(i, j,
aa + (kk - i) * i * COMPSIZE,
b + (kk - i) * j * COMPSIZE,
cc, ldc);
kk -= i;
}
int mod = i;
i = vl;
if (i <= m) {
aa = a + (m - mod - vl) * k * COMPSIZE;
cc = c + (m - mod - vl) * COMPSIZE;
i = (m >> GEMM_UNROLL_M_SHIFT);
if (i > 0) {
aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE;
cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * COMPSIZE;
do {
if (k - kk > 0) {
GEMM_KERNEL(vl, j, k - kk, dm1,
GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1,
#ifdef COMPLEX
ZERO,
#endif
aa + vl * kk * COMPSIZE,
aa + GEMM_UNROLL_M * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc,
ldc);
}
solve(vl, j,
aa + (kk - vl) * vl * COMPSIZE,
b + (kk - vl) * j * COMPSIZE,
solve(GEMM_UNROLL_M, j,
aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE,
b + (kk - GEMM_UNROLL_M) * j * COMPSIZE,
cc, ldc);
aa -= vl * k * COMPSIZE;
cc -= vl * COMPSIZE;
kk -= vl;
i += vl;
} while (i <= m);
aa -= GEMM_UNROLL_M * k * COMPSIZE;
cc -= GEMM_UNROLL_M * COMPSIZE;
kk -= GEMM_UNROLL_M;
i --;
} while (i > 0);
}
b += j * k * COMPSIZE;
kernel/riscv64/trsm_kernel_LT_rvv_v1.c
+86 -56
View File
@@ -71,6 +71,30 @@ static FLOAT dm1 = -1.;
#define GEMM_KERNEL GEMM_KERNEL_N
#endif
#if GEMM_DEFAULT_UNROLL_M == 1
#define GEMM_UNROLL_M_SHIFT 0
#endif
#if GEMM_DEFAULT_UNROLL_M == 2
#define GEMM_UNROLL_M_SHIFT 1
#endif
#if GEMM_DEFAULT_UNROLL_M == 4
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 6
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 8
#define GEMM_UNROLL_M_SHIFT 3
#endif
#if GEMM_DEFAULT_UNROLL_M == 16
#define GEMM_UNROLL_M_SHIFT 4
#endif
#if GEMM_DEFAULT_UNROLL_N == 1
#define GEMM_UNROLL_N_SHIFT 0
#endif
@@ -91,7 +115,9 @@ static FLOAT dm1 = -1.;
#define GEMM_UNROLL_N_SHIFT 4
#endif
// Optimizes the implementation in ../arm64/trsm_kernel_LT_sve.c
// Packed A is tiled by GEMM_UNROLL_M (the GEMM micro-kernel / itcopy contract),
// NOT by runtime VL: tiling by VSETVL_MAX only matches x280 and silently
// corrupts on other targets (e.g. zvl128b, unroll 8). Only solve() vectorizes.
#ifndef COMPLEX
@@ -213,10 +239,6 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
BLASLONG kk;
BLASLONG i, j;
size_t vl = VSETVL_MAX;
//fprintf(stderr, "%s , %s, m = %4ld n = %4ld k = %4ld offset = %4ld\n", __FILE__, __FUNCTION__, m, n, k, offset); // Debug
j = (n >> GEMM_UNROLL_N_SHIFT);
while (j > 0) {
@@ -225,47 +247,51 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
aa = a;
cc = c;
i = vl;
i = (m >> GEMM_UNROLL_M_SHIFT);
while (i <= m) {
while (i > 0) {
if (kk > 0) {
GEMM_KERNEL(vl, GEMM_UNROLL_N, kk, dm1,
GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1,
#ifdef COMPLEX
ZERO,
#endif
aa, b, cc, ldc);
}
solve(vl, GEMM_UNROLL_N,
aa + kk * vl * COMPSIZE,
solve(GEMM_UNROLL_M, GEMM_UNROLL_N,
aa + kk * GEMM_UNROLL_M * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
kk += vl;
i += vl;
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
kk += GEMM_UNROLL_M;
i --;
}
i = m % vl;
if (i) {
if (kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
while (i > 0) {
if (m & i) {
if (kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa, b, cc, ldc);
aa, b, cc, ldc);
}
solve(i, GEMM_UNROLL_N,
aa + kk * i * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
kk += i;
}
i >>= 1;
}
solve(i, GEMM_UNROLL_N,
aa + kk * i * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
kk += i;
}
b += GEMM_UNROLL_N * k * COMPSIZE;
@@ -283,11 +309,11 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
aa = a;
cc = c;
i = vl;
i = (m >> GEMM_UNROLL_M_SHIFT);
while (i <= m) {
while (i > 0) {
if (kk > 0) {
GEMM_KERNEL(vl, j, kk, dm1,
GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1,
#ifdef COMPLEX
ZERO,
#endif
@@ -297,37 +323,41 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
ldc);
}
solve(vl, j,
aa + kk * vl * COMPSIZE,
solve(GEMM_UNROLL_M, j,
aa + kk * GEMM_UNROLL_M * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
kk += vl;
i += vl;
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
kk += GEMM_UNROLL_M;
i --;
}
i = m % vl;
if (i) {
if (kk > 0) {
GEMM_KERNEL(i, j, kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
while (i > 0) {
if (m & i) {
if (kk > 0) {
GEMM_KERNEL(i, j, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa,
b,
cc,
ldc);
aa,
b,
cc,
ldc);
}
solve(i, j,
aa + kk * i * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
kk += i;
}
i >>= 1;
}
solve(i, j,
aa + kk * i * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
kk += i;
}
b += j * k * COMPSIZE;
kernel/riscv64/trsm_kernel_RN_rvv_v1.c
+103 -75
View File
@@ -68,6 +68,30 @@ static FLOAT dm1 = -1.;
#define GEMM_KERNEL GEMM_KERNEL_N
#endif
#if GEMM_DEFAULT_UNROLL_M == 1
#define GEMM_UNROLL_M_SHIFT 0
#endif
#if GEMM_DEFAULT_UNROLL_M == 2
#define GEMM_UNROLL_M_SHIFT 1
#endif
#if GEMM_DEFAULT_UNROLL_M == 4
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 6
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 8
#define GEMM_UNROLL_M_SHIFT 3
#endif
#if GEMM_DEFAULT_UNROLL_M == 16
#define GEMM_UNROLL_M_SHIFT 4
#endif
#if GEMM_DEFAULT_UNROLL_N == 1
#define GEMM_UNROLL_N_SHIFT 0
#endif
@@ -88,7 +112,9 @@ static FLOAT dm1 = -1.;
#define GEMM_UNROLL_N_SHIFT 4
#endif
// Optimizes the implementation in ../arm64/trsm_kernel_RN_sve.c
// Packed A is tiled by GEMM_UNROLL_M (the GEMM micro-kernel / itcopy contract),
// NOT by runtime VL: tiling by VSETVL_MAX only matches x280 and silently
// corrupts on other targets (e.g. zvl128b, unroll 8). Only solve() vectorizes.
#ifndef COMPLEX
@@ -209,11 +235,6 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
BLASLONG kk;
BLASLONG i, j;
size_t vl = VSETVL_MAX;
//fprintf(stderr, "%s , %s, m = %4ld n = %4ld k = %4ld offset = %4ld\n", __FILE__, __FUNCTION__, m, n, k, offset); // Debug
j = (n >> GEMM_UNROLL_N_SHIFT);
kk = -offset;
@@ -222,47 +243,50 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
aa = a;
cc = c;
i = vl;
i = (m >> GEMM_UNROLL_M_SHIFT);
if (i <= m) {
if (i > 0) {
do {
if (kk > 0) {
GEMM_KERNEL(vl, GEMM_UNROLL_N, kk, dm1,
if (kk > 0) {
GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa, b, cc, ldc);
}
aa, b, cc, ldc);
}
solve(vl, GEMM_UNROLL_N,
aa + kk * vl * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
solve(GEMM_UNROLL_M, GEMM_UNROLL_N,
aa + kk * GEMM_UNROLL_M * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
i += vl;
} while (i <= m);
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
i --;
} while (i > 0);
}
i = m % vl;
if (i) {
if (kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
while (i > 0) {
if (m & i) {
if (kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa, b, cc, ldc);
aa, b, cc, ldc);
}
solve(i, GEMM_UNROLL_N,
aa + kk * i * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
}
i >>= 1;
}
solve(i, GEMM_UNROLL_N,
aa + kk * i * COMPSIZE,
b + kk * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
}
kk += GEMM_UNROLL_N;
@@ -277,57 +301,61 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
while (j > 0) {
if (n & j) {
aa = a;
cc = c;
aa = a;
cc = c;
i = vl;
i = (m >> GEMM_UNROLL_M_SHIFT);
while (i <= m) {
if (kk > 0) {
GEMM_KERNEL(vl, j, kk, dm1,
while (i > 0) {
if (kk > 0) {
GEMM_KERNEL(GEMM_UNROLL_M, j, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa,
b,
cc,
ldc);
}
aa,
b,
cc,
ldc);
}
solve(vl, j,
aa + kk * vl * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
solve(GEMM_UNROLL_M, j,
aa + kk * GEMM_UNROLL_M * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
i += vl;
}
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
i --;
}
i = m % vl;
if (i) {
if (kk > 0) {
GEMM_KERNEL(i, j, kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
while (i > 0) {
if (m & i) {
if (kk > 0) {
GEMM_KERNEL(i, j, kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa,
b,
cc,
ldc);
}
aa,
b,
cc,
ldc);
}
solve(i, j,
aa + kk * i * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
solve(i, j,
aa + kk * i * COMPSIZE,
b + kk * j * COMPSIZE, cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
}
i >>= 1;
}
}
}
b += j * k * COMPSIZE;
c += j * ldc * COMPSIZE;
kk += j;
b += j * k * COMPSIZE;
c += j * ldc * COMPSIZE;
kk += j;
}
j >>= 1;
}
kernel/riscv64/trsm_kernel_RT_rvv_v1.c
+98 -66
View File
@@ -67,6 +67,30 @@ static FLOAT dm1 = -1.;
#define GEMM_KERNEL GEMM_KERNEL_N
#endif
#if GEMM_DEFAULT_UNROLL_M == 1
#define GEMM_UNROLL_M_SHIFT 0
#endif
#if GEMM_DEFAULT_UNROLL_M == 2
#define GEMM_UNROLL_M_SHIFT 1
#endif
#if GEMM_DEFAULT_UNROLL_M == 4
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 6
#define GEMM_UNROLL_M_SHIFT 2
#endif
#if GEMM_DEFAULT_UNROLL_M == 8
#define GEMM_UNROLL_M_SHIFT 3
#endif
#if GEMM_DEFAULT_UNROLL_M == 16
#define GEMM_UNROLL_M_SHIFT 4
#endif
#if GEMM_DEFAULT_UNROLL_N == 1
#define GEMM_UNROLL_N_SHIFT 0
#endif
@@ -87,7 +111,9 @@ static FLOAT dm1 = -1.;
#define GEMM_UNROLL_N_SHIFT 4
#endif
// Optimizes the implementation in ../arm64/trsm_kernel_RT_sve.c
// Packed A is tiled by GEMM_UNROLL_M (the GEMM micro-kernel / itcopy contract),
// NOT by runtime VL: tiling by VSETVL_MAX only matches x280 and silently
// corrupts on other targets (e.g. zvl128b, unroll 8). Only solve() vectorizes.
#ifndef COMPLEX
@@ -215,10 +241,6 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
FLOAT *aa, *cc;
BLASLONG kk;
size_t vl = VSETVL_MAX;
//fprintf(stderr, "%s , %s, m = %4ld n = %4ld k = %4ld offset = %4ld\n", __FILE__, __FUNCTION__, m, n, k, offset); // Debug
kk = n - offset;
c += n * ldc * COMPSIZE;
b += n * k * COMPSIZE;
@@ -234,52 +256,58 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
c -= j * ldc* COMPSIZE;
cc = c;
i = vl;
if (i <= m) {
i = (m >> GEMM_UNROLL_M_SHIFT);
if (i > 0) {
do {
if (k - kk > 0) {
GEMM_KERNEL(vl, j, k - kk, dm1,
GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1,
#ifdef COMPLEX
ZERO,
#endif
aa + vl * kk * COMPSIZE,
aa + GEMM_UNROLL_M * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc,
ldc);
}
solve(vl, j,
aa + (kk - j) * vl * COMPSIZE,
solve(GEMM_UNROLL_M, j,
aa + (kk - j) * GEMM_UNROLL_M * COMPSIZE,
b + (kk - j) * j * COMPSIZE,
cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
i += vl;
} while (i <= m);
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
i --;
} while (i > 0);
}
i = m % vl;
if (i) {
if (k - kk > 0) {
GEMM_KERNEL(i, j, k - kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
do {
if (m & i) {
if (k - kk > 0) {
GEMM_KERNEL(i, j, k - kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa + i * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc, ldc);
}
aa + i * kk * COMPSIZE,
b + j * kk * COMPSIZE,
cc, ldc);
}
solve(i, j,
aa + (kk - j) * i * COMPSIZE,
b + (kk - j) * j * COMPSIZE,
cc, ldc);
solve(i, j,
aa + (kk - j) * i * COMPSIZE,
b + (kk - j) * j * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
}
i >>= 1;
} while (i > 0);
}
kk -= j;
}
@@ -297,52 +325,56 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1,
c -= GEMM_UNROLL_N * ldc * COMPSIZE;
cc = c;
i = vl;
if (i <= m) {
do {
if (k - kk > 0) {
GEMM_KERNEL(vl, GEMM_UNROLL_N, k - kk, dm1,
i = (m >> GEMM_UNROLL_M_SHIFT);
if (i > 0) {
do {
if (k - kk > 0) {
GEMM_KERNEL(GEMM_UNROLL_M, GEMM_UNROLL_N, k - kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa + vl * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
aa + GEMM_UNROLL_M * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
solve(vl, GEMM_UNROLL_N,
aa + (kk - GEMM_UNROLL_N) * vl * COMPSIZE,
b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
solve(GEMM_UNROLL_M, GEMM_UNROLL_N,
aa + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_M * COMPSIZE,
b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += vl * k * COMPSIZE;
cc += vl * COMPSIZE;
i += vl;
} while (i <= m);
aa += GEMM_UNROLL_M * k * COMPSIZE;
cc += GEMM_UNROLL_M * COMPSIZE;
i --;
} while (i > 0);
}
i = m % vl;
if (i) {
if (k - kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,
if (m & (GEMM_UNROLL_M - 1)) {
i = (GEMM_UNROLL_M >> 1);
do {
if (m & i) {
if (k - kk > 0) {
GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,
#ifdef COMPLEX
ZERO,
ZERO,
#endif
aa + i * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
aa + i * kk * COMPSIZE,
b + GEMM_UNROLL_N * kk * COMPSIZE,
cc,
ldc);
}
solve(i, GEMM_UNROLL_N,
aa + (kk - GEMM_UNROLL_N) * i * COMPSIZE,
b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
solve(i, GEMM_UNROLL_N,
aa + (kk - GEMM_UNROLL_N) * i * COMPSIZE,
b + (kk - GEMM_UNROLL_N) * GEMM_UNROLL_N * COMPSIZE,
cc, ldc);
aa += i * k * COMPSIZE;
cc += i * COMPSIZE;
}
i >>= 1;
} while (i > 0);
}
kk -= GEMM_UNROLL_N;