phynexis-deps/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
develop
OpenBLAS/kernel/arm64/dgemm_kernel_6x8_cortexa72.S
T
Hugo MeilandandCursor 560caa56e6 Add Cortex-A72 DGEMM 6x8 microkernel with MR=6 packers and TRSM 
TARGET=CORTEXA72 previously reused the A57 8x4 path. Add a dedicated
6x8 NEON ukernel, contiguous MR=6 panel packers (stock gemm_*copy_6 is
4+2), and UNROLL_M=6-aware TRSM kernels so HPL/dtrsm does not corrupt
the heap. DTRMM falls back to generic 2x2 until a matching kernel exists.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-09 10:31:35 +02:00

1073 lines
23 KiB
ArmAsm
Raw Permalink Blame History

/*******************************************************************************
Copyright (c) 2015, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER
#include "common.h"
/* X0 X1 X2 s0 X3 x4 x5 x6 */
/*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc )*/
#define origM x0
#define origN x1
#define origK x2
#define origPA x3
#define origPB x4
#define pC x5
#define LDC x6
#define temp x7
#define counterL x8
#define counterI x9
#define counterJ x10
#define pB x11
#define pCRow0 x12
#define pCRow1 x13
#define pCRow2 x14
#define pA x15
/*
* Cortex-A72 DGEMM 6x8 microkernel (mr=6, nr=8).
*
* Register map (BLIS bli_dgemm_armv8a_asm_6x8 inspired):
* A: v0,v1,v2 = 6 packed doubles / k-step (48 B)
* B: v3,v4,v5,v6 = 8 packed doubles / k-step (64 B)
* alpha: d7 / v7.d[0]
* C col j: v(8+3*j), v(9+3*j), v(10+3*j) -> v8..v31 (24 accumulators)
*
* B/FLOP = 1.17 vs 8x4's 1.50; FLOPs/A-byte = 2.0 (NR/4).
*/
#define A_PRE_SIZE 2560
#define B_PRE_SIZE 448
#define C_PRE_SIZE 128
#define alpha0 d7
#define alphaV0 v7.d[0]
.macro INIT6x8
fmov d8, xzr
fmov d9, xzr
fmov d10, xzr
fmov d11, xzr
fmov d12, xzr
fmov d13, xzr
fmov d14, xzr
fmov d15, xzr
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
fmov d20, xzr
fmov d21, xzr
fmov d22, xzr
fmov d23, xzr
fmov d24, xzr
fmov d25, xzr
fmov d26, xzr
fmov d27, xzr
fmov d28, xzr
fmov d29, xzr
fmov d30, xzr
fmov d31, xzr
.endm
.macro KERNEL6x8_SUB
ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48
ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
prfm PLDL1KEEP, [pA, #A_PRE_SIZE]
prfm PLDL1KEEP, [pB, #B_PRE_SIZE]
fmla v8.2d, v0.2d, v3.d[0]
fmla v9.2d, v1.2d, v3.d[0]
fmla v10.2d, v2.2d, v3.d[0]
fmla v11.2d, v0.2d, v3.d[1]
fmla v12.2d, v1.2d, v3.d[1]
fmla v13.2d, v2.2d, v3.d[1]
fmla v14.2d, v0.2d, v4.d[0]
fmla v15.2d, v1.2d, v4.d[0]
fmla v16.2d, v2.2d, v4.d[0]
fmla v17.2d, v0.2d, v4.d[1]
fmla v18.2d, v1.2d, v4.d[1]
fmla v19.2d, v2.2d, v4.d[1]
fmla v20.2d, v0.2d, v5.d[0]
fmla v21.2d, v1.2d, v5.d[0]
fmla v22.2d, v2.2d, v5.d[0]
fmla v23.2d, v0.2d, v5.d[1]
fmla v24.2d, v1.2d, v5.d[1]
fmla v25.2d, v2.2d, v5.d[1]
fmla v26.2d, v0.2d, v6.d[0]
fmla v27.2d, v1.2d, v6.d[0]
fmla v28.2d, v2.2d, v6.d[0]
fmla v29.2d, v0.2d, v6.d[1]
fmla v30.2d, v1.2d, v6.d[1]
fmla v31.2d, v2.2d, v6.d[1]
.endm
.macro SAVE6x8
mov pCRow1, pCRow0
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v8.2d, alphaV0
fmla v1.2d, v9.2d, alphaV0
fmla v2.2d, v10.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v11.2d, alphaV0
fmla v1.2d, v12.2d, alphaV0
fmla v2.2d, v13.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v14.2d, alphaV0
fmla v1.2d, v15.2d, alphaV0
fmla v2.2d, v16.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v17.2d, alphaV0
fmla v1.2d, v18.2d, alphaV0
fmla v2.2d, v19.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v20.2d, alphaV0
fmla v1.2d, v21.2d, alphaV0
fmla v2.2d, v22.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v23.2d, alphaV0
fmla v1.2d, v24.2d, alphaV0
fmla v2.2d, v25.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v26.2d, alphaV0
fmla v1.2d, v27.2d, alphaV0
fmla v2.2d, v28.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
prfm PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v29.2d, alphaV0
fmla v1.2d, v30.2d, alphaV0
fmla v2.2d, v31.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
add pCRow0, pCRow0, #48
.endm
.macro INIT4x8
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
fmov d20, xzr
fmov d21, xzr
fmov d22, xzr
fmov d23, xzr
fmov d24, xzr
fmov d25, xzr
fmov d26, xzr
fmov d27, xzr
fmov d28, xzr
fmov d29, xzr
fmov d30, xzr
fmov d31, xzr
.endm
.macro KERNEL4x8_SUB
ld1 {v0.2d, v1.2d}, [pA], #32
ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
fmla v16.2d, v0.2d, v3.d[0]
fmla v17.2d, v1.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
fmla v19.2d, v1.2d, v3.d[1]
fmla v20.2d, v0.2d, v4.d[0]
fmla v21.2d, v1.2d, v4.d[0]
fmla v22.2d, v0.2d, v4.d[1]
fmla v23.2d, v1.2d, v4.d[1]
fmla v24.2d, v0.2d, v5.d[0]
fmla v25.2d, v1.2d, v5.d[0]
fmla v26.2d, v0.2d, v5.d[1]
fmla v27.2d, v1.2d, v5.d[1]
fmla v28.2d, v0.2d, v6.d[0]
fmla v29.2d, v1.2d, v6.d[0]
fmla v30.2d, v0.2d, v6.d[1]
fmla v31.2d, v1.2d, v6.d[1]
.endm
.macro SAVE4x8
mov pCRow1, pCRow0
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
fmla v1.2d, v17.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
fmla v1.2d, v19.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v20.2d, alphaV0
fmla v1.2d, v21.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v22.2d, alphaV0
fmla v1.2d, v23.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v24.2d, alphaV0
fmla v1.2d, v25.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v26.2d, alphaV0
fmla v1.2d, v27.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v28.2d, alphaV0
fmla v1.2d, v29.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v30.2d, alphaV0
fmla v1.2d, v31.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
add pCRow0, pCRow0, #32
.endm
.macro INIT2x8
fmov d16, xzr
fmov d18, xzr
fmov d20, xzr
fmov d22, xzr
fmov d24, xzr
fmov d26, xzr
fmov d28, xzr
fmov d30, xzr
.endm
.macro KERNEL2x8_SUB
ld1 {v0.2d}, [pA], #16
ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
fmla v16.2d, v0.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
fmla v20.2d, v0.2d, v4.d[0]
fmla v22.2d, v0.2d, v4.d[1]
fmla v24.2d, v0.2d, v5.d[0]
fmla v26.2d, v0.2d, v5.d[1]
fmla v28.2d, v0.2d, v6.d[0]
fmla v30.2d, v0.2d, v6.d[1]
.endm
.macro SAVE2x8
mov pCRow1, pCRow0
ldr q0, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v20.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v22.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v24.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v26.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v28.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v30.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
add pCRow0, pCRow0, #16
.endm
.macro INIT1x8
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
fmov d20, xzr
fmov d21, xzr
fmov d22, xzr
fmov d23, xzr
.endm
.macro KERNEL1x8_SUB
ldr d0, [pA], #8
ld1 {v3.2d, v4.2d, v5.2d, v6.2d}, [pB], #64
fmadd d16, d0, d3, d16
fmul d1, d0, v3.d[1]
fadd d17, d17, d1
fmadd d18, d0, d4, d18
fmul d1, d0, v4.d[1]
fadd d19, d19, d1
fmadd d20, d0, d5, d20
fmul d1, d0, v5.d[1]
fadd d21, d21, d1
fmadd d22, d0, d6, d22
fmul d1, d0, v6.d[1]
fadd d23, d23, d1
.endm
.macro SAVE1x8
mov pCRow1, pCRow0
ldr d0, [pCRow1]
fmadd d0, d16, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d17, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d18, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d19, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d20, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d21, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d22, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d23, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
add pCRow0, pCRow0, #8
.endm
.macro INIT6x4
fmov d8, xzr
fmov d9, xzr
fmov d10, xzr
fmov d11, xzr
fmov d12, xzr
fmov d13, xzr
fmov d14, xzr
fmov d15, xzr
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
.endm
.macro KERNEL6x4_SUB
ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48
ld1 {v3.2d, v4.2d}, [pB], #32
fmla v8.2d, v0.2d, v3.d[0]
fmla v9.2d, v1.2d, v3.d[0]
fmla v10.2d, v2.2d, v3.d[0]
fmla v11.2d, v0.2d, v3.d[1]
fmla v12.2d, v1.2d, v3.d[1]
fmla v13.2d, v2.2d, v3.d[1]
fmla v14.2d, v0.2d, v4.d[0]
fmla v15.2d, v1.2d, v4.d[0]
fmla v16.2d, v2.2d, v4.d[0]
fmla v17.2d, v0.2d, v4.d[1]
fmla v18.2d, v1.2d, v4.d[1]
fmla v19.2d, v2.2d, v4.d[1]
.endm
.macro SAVE6x4
mov pCRow1, pCRow0
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v8.2d, alphaV0
fmla v1.2d, v9.2d, alphaV0
fmla v2.2d, v10.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v11.2d, alphaV0
fmla v1.2d, v12.2d, alphaV0
fmla v2.2d, v13.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v14.2d, alphaV0
fmla v1.2d, v15.2d, alphaV0
fmla v2.2d, v16.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v17.2d, alphaV0
fmla v1.2d, v18.2d, alphaV0
fmla v2.2d, v19.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
add pCRow0, pCRow0, #48
.endm
.macro INIT6x2
fmov d8, xzr
fmov d9, xzr
fmov d10, xzr
fmov d11, xzr
fmov d12, xzr
fmov d13, xzr
.endm
.macro KERNEL6x2_SUB
ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48
ld1 {v3.2d}, [pB], #16
fmla v8.2d, v0.2d, v3.d[0]
fmla v9.2d, v1.2d, v3.d[0]
fmla v10.2d, v2.2d, v3.d[0]
fmla v11.2d, v0.2d, v3.d[1]
fmla v12.2d, v1.2d, v3.d[1]
fmla v13.2d, v2.2d, v3.d[1]
.endm
.macro SAVE6x2
mov pCRow1, pCRow0
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v8.2d, alphaV0
fmla v1.2d, v9.2d, alphaV0
fmla v2.2d, v10.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
fmla v0.2d, v11.2d, alphaV0
fmla v1.2d, v12.2d, alphaV0
fmla v2.2d, v13.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow1]
add pCRow0, pCRow0, #48
.endm
.macro INIT6x1
fmov d8, xzr
fmov d9, xzr
fmov d10, xzr
.endm
.macro KERNEL6x1_SUB
ld1 {v0.2d, v1.2d, v2.2d}, [pA], #48
ldr d3, [pB], #8
fmla v8.2d, v0.2d, v3.d[0]
fmla v9.2d, v1.2d, v3.d[0]
fmla v10.2d, v2.2d, v3.d[0]
.endm
.macro SAVE6x1
ld1 {v0.2d, v1.2d, v2.2d}, [pCRow0]
fmla v0.2d, v8.2d, alphaV0
fmla v1.2d, v9.2d, alphaV0
fmla v2.2d, v10.2d, alphaV0
st1 {v0.2d, v1.2d, v2.2d}, [pCRow0]
add pCRow0, pCRow0, #48
.endm
.macro INIT4x4
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
fmov d20, xzr
fmov d21, xzr
fmov d22, xzr
fmov d23, xzr
.endm
.macro KERNEL4x4_SUB
ld1 {v0.2d, v1.2d}, [pA], #32
ld1 {v3.2d, v4.2d}, [pB], #32
fmla v16.2d, v0.2d, v3.d[0]
fmla v17.2d, v1.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
fmla v19.2d, v1.2d, v3.d[1]
fmla v20.2d, v0.2d, v4.d[0]
fmla v21.2d, v1.2d, v4.d[0]
fmla v22.2d, v0.2d, v4.d[1]
fmla v23.2d, v1.2d, v4.d[1]
.endm
.macro SAVE4x4
mov pCRow1, pCRow0
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
fmla v1.2d, v17.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
fmla v1.2d, v19.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v20.2d, alphaV0
fmla v1.2d, v21.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v22.2d, alphaV0
fmla v1.2d, v23.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow0, pCRow0, #32
.endm
.macro INIT2x4
fmov d16, xzr
fmov d18, xzr
fmov d20, xzr
fmov d22, xzr
.endm
.macro KERNEL2x4_SUB
ld1 {v0.2d}, [pA], #16
ld1 {v3.2d, v4.2d}, [pB], #32
fmla v16.2d, v0.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
fmla v20.2d, v0.2d, v4.d[0]
fmla v22.2d, v0.2d, v4.d[1]
.endm
.macro SAVE2x4
mov pCRow1, pCRow0
ldr q0, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v20.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v22.2d, alphaV0
str q0, [pCRow1]
add pCRow0, pCRow0, #16
.endm
.macro INIT1x4
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
.endm
.macro KERNEL1x4_SUB
ldr d0, [pA], #8
ld1 {v3.2d, v4.2d}, [pB], #32
fmadd d16, d0, d3, d16
fmul d1, d0, v3.d[1]
fadd d17, d17, d1
fmadd d18, d0, d4, d18
fmul d1, d0, v4.d[1]
fadd d19, d19, d1
.endm
.macro SAVE1x4
mov pCRow1, pCRow0
ldr d0, [pCRow1]
fmadd d0, d16, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d17, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d18, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d19, alpha0, d0
str d0, [pCRow1]
add pCRow0, pCRow0, #8
.endm
.macro INIT4x2
fmov d16, xzr
fmov d17, xzr
fmov d18, xzr
fmov d19, xzr
.endm
.macro KERNEL4x2_SUB
ld1 {v0.2d, v1.2d}, [pA], #32
ld1 {v3.2d}, [pB], #16
fmla v16.2d, v0.2d, v3.d[0]
fmla v17.2d, v1.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
fmla v19.2d, v1.2d, v3.d[1]
.endm
.macro SAVE4x2
mov pCRow1, pCRow0
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
fmla v1.2d, v17.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow1, pCRow1, LDC
ld1 {v0.2d, v1.2d}, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
fmla v1.2d, v19.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow1]
add pCRow0, pCRow0, #32
.endm
.macro INIT2x2
fmov d16, xzr
fmov d18, xzr
.endm
.macro KERNEL2x2_SUB
ld1 {v0.2d}, [pA], #16
ld1 {v3.2d}, [pB], #16
fmla v16.2d, v0.2d, v3.d[0]
fmla v18.2d, v0.2d, v3.d[1]
.endm
.macro SAVE2x2
mov pCRow1, pCRow0
ldr q0, [pCRow1]
fmla v0.2d, v16.2d, alphaV0
str q0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr q0, [pCRow1]
fmla v0.2d, v18.2d, alphaV0
str q0, [pCRow1]
add pCRow0, pCRow0, #16
.endm
.macro INIT1x2
fmov d16, xzr
fmov d17, xzr
.endm
.macro KERNEL1x2_SUB
ldr d0, [pA], #8
ld1 {v3.2d}, [pB], #16
fmadd d16, d0, d3, d16
fmul d1, d0, v3.d[1]
fadd d17, d17, d1
.endm
.macro SAVE1x2
mov pCRow1, pCRow0
ldr d0, [pCRow1]
fmadd d0, d16, alpha0, d0
str d0, [pCRow1]
add pCRow1, pCRow1, LDC
ldr d0, [pCRow1]
fmadd d0, d17, alpha0, d0
str d0, [pCRow1]
add pCRow0, pCRow0, #8
.endm
.macro INIT4x1
fmov d16, xzr
fmov d17, xzr
.endm
.macro KERNEL4x1_SUB
ld1 {v0.2d, v1.2d}, [pA], #32
ldr d3, [pB], #8
fmla v16.2d, v0.2d, v3.d[0]
fmla v17.2d, v1.2d, v3.d[0]
.endm
.macro SAVE4x1
ld1 {v0.2d, v1.2d}, [pCRow0]
fmla v0.2d, v16.2d, alphaV0
fmla v1.2d, v17.2d, alphaV0
st1 {v0.2d, v1.2d}, [pCRow0]
add pCRow0, pCRow0, #32
.endm
.macro INIT2x1
fmov d16, xzr
.endm
.macro KERNEL2x1_SUB
ld1 {v0.2d}, [pA], #16
ldr d3, [pB], #8
fmla v16.2d, v0.2d, v3.d[0]
.endm
.macro SAVE2x1
ldr q0, [pCRow0]
fmla v0.2d, v16.2d, alphaV0
str q0, [pCRow0]
add pCRow0, pCRow0, #16
.endm
.macro INIT1x1
fmov d16, xzr
.endm
.macro KERNEL1x1_SUB
ldr d0, [pA], #8
ldr d3, [pB], #8
fmadd d16, d0, d3, d16
.endm
.macro SAVE1x1
ldr d0, [pCRow0]
fmadd d0, d16, alpha0, d0
str d0, [pCRow0]
add pCRow0, pCRow0, #8
.endm
PROLOGUE
.align 5
add sp, sp, #-(11 * 16)
stp d8, d9, [sp, #(0 * 16)]
stp d10, d11, [sp, #(1 * 16)]
stp d12, d13, [sp, #(2 * 16)]
stp d14, d15, [sp, #(3 * 16)]
stp d16, d17, [sp, #(4 * 16)]
stp x18, x19, [sp, #(5 * 16)]
stp x20, x21, [sp, #(6 * 16)]
stp x22, x23, [sp, #(7 * 16)]
stp x24, x25, [sp, #(8 * 16)]
stp x26, x27, [sp, #(9 * 16)]
str x28, [sp, #(10 * 16)]
fmov alpha0, d0
lsl LDC, LDC, #3
mov counterJ, origN
asr counterJ, counterJ, #3
cbz counterJ, .L6x8_L4_BEGIN
.L6x8_L8_BEGIN:
mov pCRow0, pC
add pC, pC, LDC, lsl #3
mov pA, origPA
mov temp, #6
udiv counterI, origM, temp
cbz counterI, .L6x8_L8_Mrem
.L6x8_L8_M6:
INIT6x8
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L8_M6s
.L6x8_L8_M6l:
KERNEL6x8_SUB
subs counterL, counterL, #1
bgt .L6x8_L8_M6l
.L6x8_L8_M6s:
SAVE6x8
subs counterI, counterI, #1
bgt .L6x8_L8_M6
.L6x8_L8_Mrem:
mov temp, #6
udiv counterI, origM, temp
msub counterI, counterI, temp, origM
cmp counterI, #4
blt .L6x8_L8_M2c
INIT4x8
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L8_M4s
.L6x8_L8_M4l:
KERNEL4x8_SUB
subs counterL, counterL, #1
bgt .L6x8_L8_M4l
.L6x8_L8_M4s:
SAVE4x8
sub counterI, counterI, #4
.L6x8_L8_M2c:
cmp counterI, #2
blt .L6x8_L8_M1c
INIT2x8
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L8_M2s
.L6x8_L8_M2l:
KERNEL2x8_SUB
subs counterL, counterL, #1
bgt .L6x8_L8_M2l
.L6x8_L8_M2s:
SAVE2x8
sub counterI, counterI, #2
.L6x8_L8_M1c:
cbz counterI, .L6x8_L8_END
INIT1x8
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L8_M1s
.L6x8_L8_M1l:
KERNEL1x8_SUB
subs counterL, counterL, #1
bgt .L6x8_L8_M1l
.L6x8_L8_M1s:
SAVE1x8
.L6x8_L8_END:
lsl temp, origK, #6 /* + K*8*8 bytes */
add origPB, origPB, temp
subs counterJ, counterJ, #1
bgt .L6x8_L8_BEGIN
.L6x8_L4_BEGIN:
tst origN, #7
beq .L6x8_L999
tst origN, #4
beq .L6x8_L2_BEGIN
mov pCRow0, pC
add pC, pC, LDC, lsl #2
mov pA, origPA
mov temp, #6
udiv counterI, origM, temp
cbz counterI, .L6x8_L4_Mrem
.L6x8_L4_M6:
INIT6x4
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L4_M6s
.L6x8_L4_M6l:
KERNEL6x4_SUB
subs counterL, counterL, #1
bgt .L6x8_L4_M6l
.L6x8_L4_M6s:
SAVE6x4
subs counterI, counterI, #1
bgt .L6x8_L4_M6
.L6x8_L4_Mrem:
mov temp, #6
udiv counterI, origM, temp
msub counterI, counterI, temp, origM
cmp counterI, #4
blt .L6x8_L4_M2c
INIT4x4
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L4_M4s
.L6x8_L4_M4l:
KERNEL4x4_SUB
subs counterL, counterL, #1
bgt .L6x8_L4_M4l
.L6x8_L4_M4s:
SAVE4x4
sub counterI, counterI, #4
.L6x8_L4_M2c:
cmp counterI, #2
blt .L6x8_L4_M1c
INIT2x4
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L4_M2s
.L6x8_L4_M2l:
KERNEL2x4_SUB
subs counterL, counterL, #1
bgt .L6x8_L4_M2l
.L6x8_L4_M2s:
SAVE2x4
sub counterI, counterI, #2
.L6x8_L4_M1c:
cbz counterI, .L6x8_L4_END
INIT1x4
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L4_M1s
.L6x8_L4_M1l:
KERNEL1x4_SUB
subs counterL, counterL, #1
bgt .L6x8_L4_M1l
.L6x8_L4_M1s:
SAVE1x4
.L6x8_L4_END:
add origPB, origPB, origK, lsl #5 /* +K*4*8 */
.L6x8_L2_BEGIN:
tst origN, #2
beq .L6x8_L1_BEGIN
mov pCRow0, pC
add pC, pC, LDC, lsl #1
mov pA, origPA
mov temp, #6
udiv counterI, origM, temp
cbz counterI, .L6x8_L2_Mrem
.L6x8_L2_M6:
INIT6x2
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L2_M6s
.L6x8_L2_M6l:
KERNEL6x2_SUB
subs counterL, counterL, #1
bgt .L6x8_L2_M6l
.L6x8_L2_M6s:
SAVE6x2
subs counterI, counterI, #1
bgt .L6x8_L2_M6
.L6x8_L2_Mrem:
mov temp, #6
udiv counterI, origM, temp
msub counterI, counterI, temp, origM
cmp counterI, #4
blt .L6x8_L2_M2c
INIT4x2
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L2_M4s
.L6x8_L2_M4l:
KERNEL4x2_SUB
subs counterL, counterL, #1
bgt .L6x8_L2_M4l
.L6x8_L2_M4s:
SAVE4x2
sub counterI, counterI, #4
.L6x8_L2_M2c:
cmp counterI, #2
blt .L6x8_L2_M1c
INIT2x2
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L2_M2s
.L6x8_L2_M2l:
KERNEL2x2_SUB
subs counterL, counterL, #1
bgt .L6x8_L2_M2l
.L6x8_L2_M2s:
SAVE2x2
sub counterI, counterI, #2
.L6x8_L2_M1c:
cbz counterI, .L6x8_L2_END
INIT1x2
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L2_M1s
.L6x8_L2_M1l:
KERNEL1x2_SUB
subs counterL, counterL, #1
bgt .L6x8_L2_M1l
.L6x8_L2_M1s:
SAVE1x2
.L6x8_L2_END:
add origPB, origPB, origK, lsl #4
.L6x8_L1_BEGIN:
tst origN, #1
beq .L6x8_L999
mov pCRow0, pC
mov pA, origPA
mov temp, #6
udiv counterI, origM, temp
cbz counterI, .L6x8_L1_Mrem
.L6x8_L1_M6:
INIT6x1
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L1_M6s
.L6x8_L1_M6l:
KERNEL6x1_SUB
subs counterL, counterL, #1
bgt .L6x8_L1_M6l
.L6x8_L1_M6s:
SAVE6x1
subs counterI, counterI, #1
bgt .L6x8_L1_M6
.L6x8_L1_Mrem:
mov temp, #6
udiv counterI, origM, temp
msub counterI, counterI, temp, origM
cmp counterI, #4
blt .L6x8_L1_M2c
INIT4x1
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L1_M4s
.L6x8_L1_M4l:
KERNEL4x1_SUB
subs counterL, counterL, #1
bgt .L6x8_L1_M4l
.L6x8_L1_M4s:
SAVE4x1
sub counterI, counterI, #4
.L6x8_L1_M2c:
cmp counterI, #2
blt .L6x8_L1_M1c
INIT2x1
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L1_M2s
.L6x8_L1_M2l:
KERNEL2x1_SUB
subs counterL, counterL, #1
bgt .L6x8_L1_M2l
.L6x8_L1_M2s:
SAVE2x1
sub counterI, counterI, #2
.L6x8_L1_M1c:
cbz counterI, .L6x8_L999
INIT1x1
mov pB, origPB
mov counterL, origK
cbz counterL, .L6x8_L1_M1s
.L6x8_L1_M1l:
KERNEL1x1_SUB
subs counterL, counterL, #1
bgt .L6x8_L1_M1l
.L6x8_L1_M1s:
SAVE1x1
.L6x8_L999:
mov x0, #0
ldp d8, d9, [sp, #(0 * 16)]
ldp d10, d11, [sp, #(1 * 16)]
ldp d12, d13, [sp, #(2 * 16)]
ldp d14, d15, [sp, #(3 * 16)]
ldp d16, d17, [sp, #(4 * 16)]
ldp x18, x19, [sp, #(5 * 16)]
ldp x20, x21, [sp, #(6 * 16)]
ldp x22, x23, [sp, #(7 * 16)]
ldp x24, x25, [sp, #(8 * 16)]
ldp x26, x27, [sp, #(9 * 16)]
ldr x28, [sp, #(10 * 16)]
add sp, sp, #(11 * 16)
ret
EPILOGUE
Reference in New Issue View Git Blame Copy Permalink