phynexis-deps/OpenBLAS
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
develop
OpenBLAS/kernel/generic/neg_tcopy_6.c
T
Hugo MeilandandCursor 2b69faff24 Add generic neg_tcopy_6 for DGEMM_UNROLL_M=6 
DYNAMIC_ARCH builds CORTEXA72 as a separate kernel and pull
dneg_tcopy from generic/neg_tcopy_$(DGEMM_UNROLL_M).c. Width 6 was
missing (only 1/2/4/8/16 existed), which broke the arm64 Graviton
Cirun and Azure DYNAMIC_ARM64 jobs.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-09 12:16:01 +02:00

446 lines
11 KiB
C
Raw Permalink Blame History

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/
#include <stdio.h>
#include "common.h"
/* Contiguous MR=6 negating transpose pack (needed when DGEMM_UNROLL_M=6).
* Layout matches the Goto-style panel packing of neg_tcopy_4/8: full
* 6-row panels first, then m remainders of 4 / 2 / 1. */
int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){
BLASLONG i, j;
FLOAT *a_offset, *a_offset1, *a_offset2, *a_offset3, *a_offset4;
FLOAT *a_offset5, *a_offset6;
FLOAT *b_offset, *b_offset1, *b_offset2, *b_offset3;
FLOAT ctemp1, ctemp2, ctemp3, ctemp4;
FLOAT ctemp5, ctemp6, ctemp7, ctemp8;
FLOAT ctemp9, ctemp10, ctemp11, ctemp12;
FLOAT ctemp13, ctemp14, ctemp15, ctemp16;
FLOAT ctemp17, ctemp18, ctemp19, ctemp20;
FLOAT ctemp21, ctemp22, ctemp23, ctemp24;
a_offset = a;
b_offset = b;
b_offset2 = b + m * (n & ~3);
b_offset3 = b + m * (n & ~1);
j = (m / 6);
if (j > 0){
do{
a_offset1 = a_offset;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset4 = a_offset3 + lda;
a_offset5 = a_offset4 + lda;
a_offset6 = a_offset5 + lda;
a_offset += 6 * lda;
b_offset1 = b_offset;
b_offset += 24;
i = (n >> 2);
if (i > 0){
do{
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset1 + 2);
ctemp4 = *(a_offset1 + 3);
ctemp5 = *(a_offset2 + 0);
ctemp6 = *(a_offset2 + 1);
ctemp7 = *(a_offset2 + 2);
ctemp8 = *(a_offset2 + 3);
ctemp9 = *(a_offset3 + 0);
ctemp10 = *(a_offset3 + 1);
ctemp11 = *(a_offset3 + 2);
ctemp12 = *(a_offset3 + 3);
ctemp13 = *(a_offset4 + 0);
ctemp14 = *(a_offset4 + 1);
ctemp15 = *(a_offset4 + 2);
ctemp16 = *(a_offset4 + 3);
ctemp17 = *(a_offset5 + 0);
ctemp18 = *(a_offset5 + 1);
ctemp19 = *(a_offset5 + 2);
ctemp20 = *(a_offset5 + 3);
ctemp21 = *(a_offset6 + 0);
ctemp22 = *(a_offset6 + 1);
ctemp23 = *(a_offset6 + 2);
ctemp24 = *(a_offset6 + 3);
a_offset1 += 4;
a_offset2 += 4;
a_offset3 += 4;
a_offset4 += 4;
a_offset5 += 4;
a_offset6 += 4;
*(b_offset1 + 0) = -ctemp1;
*(b_offset1 + 1) = -ctemp2;
*(b_offset1 + 2) = -ctemp3;
*(b_offset1 + 3) = -ctemp4;
*(b_offset1 + 4) = -ctemp5;
*(b_offset1 + 5) = -ctemp6;
*(b_offset1 + 6) = -ctemp7;
*(b_offset1 + 7) = -ctemp8;
*(b_offset1 + 8) = -ctemp9;
*(b_offset1 + 9) = -ctemp10;
*(b_offset1 + 10) = -ctemp11;
*(b_offset1 + 11) = -ctemp12;
*(b_offset1 + 12) = -ctemp13;
*(b_offset1 + 13) = -ctemp14;
*(b_offset1 + 14) = -ctemp15;
*(b_offset1 + 15) = -ctemp16;
*(b_offset1 + 16) = -ctemp17;
*(b_offset1 + 17) = -ctemp18;
*(b_offset1 + 18) = -ctemp19;
*(b_offset1 + 19) = -ctemp20;
*(b_offset1 + 20) = -ctemp21;
*(b_offset1 + 21) = -ctemp22;
*(b_offset1 + 22) = -ctemp23;
*(b_offset1 + 23) = -ctemp24;
b_offset1 += m * 4;
i --;
}while(i > 0);
}
if (n & 2) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset2 + 0);
ctemp4 = *(a_offset2 + 1);
ctemp5 = *(a_offset3 + 0);
ctemp6 = *(a_offset3 + 1);
ctemp7 = *(a_offset4 + 0);
ctemp8 = *(a_offset4 + 1);
ctemp9 = *(a_offset5 + 0);
ctemp10 = *(a_offset5 + 1);
ctemp11 = *(a_offset6 + 0);
ctemp12 = *(a_offset6 + 1);
a_offset1 += 2;
a_offset2 += 2;
a_offset3 += 2;
a_offset4 += 2;
a_offset5 += 2;
a_offset6 += 2;
*(b_offset2 + 0) = -ctemp1;
*(b_offset2 + 1) = -ctemp2;
*(b_offset2 + 2) = -ctemp3;
*(b_offset2 + 3) = -ctemp4;
*(b_offset2 + 4) = -ctemp5;
*(b_offset2 + 5) = -ctemp6;
*(b_offset2 + 6) = -ctemp7;
*(b_offset2 + 7) = -ctemp8;
*(b_offset2 + 8) = -ctemp9;
*(b_offset2 + 9) = -ctemp10;
*(b_offset2 + 10) = -ctemp11;
*(b_offset2 + 11) = -ctemp12;
b_offset2 += 12;
}
if (n & 1) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset2 + 0);
ctemp3 = *(a_offset3 + 0);
ctemp4 = *(a_offset4 + 0);
ctemp5 = *(a_offset5 + 0);
ctemp6 = *(a_offset6 + 0);
*(b_offset3 + 0) = -ctemp1;
*(b_offset3 + 1) = -ctemp2;
*(b_offset3 + 2) = -ctemp3;
*(b_offset3 + 3) = -ctemp4;
*(b_offset3 + 4) = -ctemp5;
*(b_offset3 + 5) = -ctemp6;
b_offset3 += 6;
}
j--;
}while(j > 0);
}
/* Remainders after m/6 panels are 0..5 — cascade 4, then 2, then 1.
* Do not use (m & 2)/(m & 1) on the original m: those bits are not the
* leftover after a non-power-of-2 main panel. */
j = m % 6;
if (j >= 4){
a_offset1 = a_offset;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset4 = a_offset3 + lda;
a_offset += 4 * lda;
b_offset1 = b_offset;
b_offset += 16;
i = (n >> 2);
if (i > 0){
do{
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset1 + 2);
ctemp4 = *(a_offset1 + 3);
ctemp5 = *(a_offset2 + 0);
ctemp6 = *(a_offset2 + 1);
ctemp7 = *(a_offset2 + 2);
ctemp8 = *(a_offset2 + 3);
ctemp9 = *(a_offset3 + 0);
ctemp10 = *(a_offset3 + 1);
ctemp11 = *(a_offset3 + 2);
ctemp12 = *(a_offset3 + 3);
ctemp13 = *(a_offset4 + 0);
ctemp14 = *(a_offset4 + 1);
ctemp15 = *(a_offset4 + 2);
ctemp16 = *(a_offset4 + 3);
a_offset1 += 4;
a_offset2 += 4;
a_offset3 += 4;
a_offset4 += 4;
*(b_offset1 + 0) = -ctemp1;
*(b_offset1 + 1) = -ctemp2;
*(b_offset1 + 2) = -ctemp3;
*(b_offset1 + 3) = -ctemp4;
*(b_offset1 + 4) = -ctemp5;
*(b_offset1 + 5) = -ctemp6;
*(b_offset1 + 6) = -ctemp7;
*(b_offset1 + 7) = -ctemp8;
*(b_offset1 + 8) = -ctemp9;
*(b_offset1 + 9) = -ctemp10;
*(b_offset1 + 10) = -ctemp11;
*(b_offset1 + 11) = -ctemp12;
*(b_offset1 + 12) = -ctemp13;
*(b_offset1 + 13) = -ctemp14;
*(b_offset1 + 14) = -ctemp15;
*(b_offset1 + 15) = -ctemp16;
b_offset1 += m * 4;
i --;
}while(i > 0);
}
if (n & 2) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset2 + 0);
ctemp4 = *(a_offset2 + 1);
ctemp5 = *(a_offset3 + 0);
ctemp6 = *(a_offset3 + 1);
ctemp7 = *(a_offset4 + 0);
ctemp8 = *(a_offset4 + 1);
a_offset1 += 2;
a_offset2 += 2;
a_offset3 += 2;
a_offset4 += 2;
*(b_offset2 + 0) = -ctemp1;
*(b_offset2 + 1) = -ctemp2;
*(b_offset2 + 2) = -ctemp3;
*(b_offset2 + 3) = -ctemp4;
*(b_offset2 + 4) = -ctemp5;
*(b_offset2 + 5) = -ctemp6;
*(b_offset2 + 6) = -ctemp7;
*(b_offset2 + 7) = -ctemp8;
b_offset2 += 8;
}
if (n & 1) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset2 + 0);
ctemp3 = *(a_offset3 + 0);
ctemp4 = *(a_offset4 + 0);
*(b_offset3 + 0) = -ctemp1;
*(b_offset3 + 1) = -ctemp2;
*(b_offset3 + 2) = -ctemp3;
*(b_offset3 + 3) = -ctemp4;
b_offset3 += 4;
}
j -= 4;
}
if (j >= 2){
a_offset1 = a_offset;
a_offset2 = a_offset1 + lda;
a_offset += 2 * lda;
b_offset1 = b_offset;
b_offset += 8;
i = (n >> 2);
if (i > 0){
do{
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset1 + 2);
ctemp4 = *(a_offset1 + 3);
ctemp5 = *(a_offset2 + 0);
ctemp6 = *(a_offset2 + 1);
ctemp7 = *(a_offset2 + 2);
ctemp8 = *(a_offset2 + 3);
a_offset1 += 4;
a_offset2 += 4;
*(b_offset1 + 0) = -ctemp1;
*(b_offset1 + 1) = -ctemp2;
*(b_offset1 + 2) = -ctemp3;
*(b_offset1 + 3) = -ctemp4;
*(b_offset1 + 4) = -ctemp5;
*(b_offset1 + 5) = -ctemp6;
*(b_offset1 + 6) = -ctemp7;
*(b_offset1 + 7) = -ctemp8;
b_offset1 += m * 4;
i --;
}while(i > 0);
}
if (n & 2) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset2 + 0);
ctemp4 = *(a_offset2 + 1);
a_offset1 += 2;
a_offset2 += 2;
*(b_offset2 + 0) = -ctemp1;
*(b_offset2 + 1) = -ctemp2;
*(b_offset2 + 2) = -ctemp3;
*(b_offset2 + 3) = -ctemp4;
b_offset2 += 4;
}
if (n & 1) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset2 + 0);
*(b_offset3 + 0) = -ctemp1;
*(b_offset3 + 1) = -ctemp2;
b_offset3 += 2;
}
j -= 2;
}
if (j >= 1){
a_offset1 = a_offset;
b_offset1 = b_offset;
i = (n >> 2);
if (i > 0){
do{
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
ctemp3 = *(a_offset1 + 2);
ctemp4 = *(a_offset1 + 3);
a_offset1 += 4;
*(b_offset1 + 0) = -ctemp1;
*(b_offset1 + 1) = -ctemp2;
*(b_offset1 + 2) = -ctemp3;
*(b_offset1 + 3) = -ctemp4;
b_offset1 += 4 * m;
i --;
}while(i > 0);
}
if (n & 2) {
ctemp1 = *(a_offset1 + 0);
ctemp2 = *(a_offset1 + 1);
a_offset1 += 2;
*(b_offset2 + 0) = -ctemp1;
*(b_offset2 + 1) = -ctemp2;
}
if (n & 1) {
ctemp1 = *(a_offset1 + 0);
*(b_offset3 + 0) = -ctemp1;
}
}
return 0;
}
Reference in New Issue View Git Blame Copy Permalink