Update version to 0.3.18

Merge from develop for 0.3.18

.travis.yml

@@ -1,33 +1,38 @@
 # XXX: Precise is already deprecated, new default is Trusty.
 # https://blog.travis-ci.com/2017-07-11-trusty-as-default-linux-is-coming
-dist: precise
+dist: focal
 sudo: true
 language: c
 
 matrix:
   include:
     - &test-ubuntu
-      os: linux
+#      os: linux
       compiler: gcc
       addons:
         apt:
           packages:
             - gfortran
+#      before_script: &common-before
+#        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32"
+#      script:
+#        - make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+#        - make -C test $COMMON_FLAGS $BTYPE
+#        - make -C ctest $COMMON_FLAGS $BTYPE
+#        - make -C utest $COMMON_FLAGS $BTYPE
+#      env:
+#        - TARGET_BOX=LINUX64
+#        - BTYPE="BINARY=64"
+#
+#    - <<: *test-ubuntu
+      os: linux-ppc64le
       before_script: &common-before
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32"
+        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=POWER8 NUM_THREADS=32"
       script:
         - make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
         - make -C test $COMMON_FLAGS $BTYPE
         - make -C ctest $COMMON_FLAGS $BTYPE
         - make -C utest $COMMON_FLAGS $BTYPE
-      env:
-        - TARGET_BOX=LINUX64
-        - BTYPE="BINARY=64"
-
-    - <<: *test-ubuntu
-      os: linux-ppc64le
-      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=POWER8 NUM_THREADS=32"
       env:
         # for matrix annotation only
         - TARGET_BOX=PPC64LE_LINUX
@@ -55,38 +60,38 @@ matrix:
         - TARGET_BOX=IBMZ_LINUX
         - BTYPE="BINARY=64 USE_OPENMP=0 CC=clang"
 
-    - <<: *test-ubuntu
-      env:
-        - TARGET_BOX=LINUX64
-        - BTYPE="BINARY=64 USE_OPENMP=1"
-
-    - <<: *test-ubuntu
-      env:
-        - TARGET_BOX=LINUX64
-        - BTYPE="BINARY=64 INTERFACE64=1"
-
-    - <<: *test-ubuntu
-      compiler: clang
-      env:
-        - TARGET_BOX=LINUX64
-        - BTYPE="BINARY=64 CC=clang"
-
-    - <<: *test-ubuntu
-      compiler: clang
-      env:
-        - TARGET_BOX=LINUX64
-        - BTYPE="BINARY=64 INTERFACE64=1 CC=clang"
-
-    - <<: *test-ubuntu
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - gfortran-multilib
-      env:
-        - TARGET_BOX=LINUX32
-        - BTYPE="BINARY=32"
-
+#    - <<: *test-ubuntu
+#      env:
+#        - TARGET_BOX=LINUX64
+#        - BTYPE="BINARY=64 USE_OPENMP=1"
+#
+#    - <<: *test-ubuntu
+#      env:
+#        - TARGET_BOX=LINUX64
+#        - BTYPE="BINARY=64 INTERFACE64=1"
+#
+#    - <<: *test-ubuntu
+#      compiler: clang
+#      env:
+#        - TARGET_BOX=LINUX64
+#        - BTYPE="BINARY=64 CC=clang"
+#
+#    - <<: *test-ubuntu
+#      compiler: clang
+#      env:
+#        - TARGET_BOX=LINUX64
+#        - BTYPE="BINARY=64 INTERFACE64=1 CC=clang"
+#
+#    - <<: *test-ubuntu
+#      addons:
+#        apt:
+#          packages:
+#            - gcc-multilib
+#            - gfortran-multilib
+#      env:
+#        - TARGET_BOX=LINUX32
+#        - BTYPE="BINARY=32"
+#
     - os: linux
       arch: ppc64le
       dist: bionic
@@ -121,47 +126,47 @@ matrix:
         # for matrix annotation only
         - TARGET_BOX=PPC64LE_LINUX_P9
 
-    - os: linux
-      compiler: gcc
-      addons:
-        apt:
-          packages:
-            - binutils-mingw-w64-x86-64
-            - gcc-mingw-w64-x86-64
-            - gfortran-mingw-w64-x86-64
-      before_script: *common-before
-      script:
-        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
-      env:
-        - TARGET_BOX=WIN64
-        - BTYPE="BINARY=64 HOSTCC=gcc CC=x86_64-w64-mingw32-gcc FC=x86_64-w64-mingw32-gfortran"
-
+#    - os: linux
+#      compiler: gcc
+#      addons:
+#        apt:
+#          packages:
+#            - binutils-mingw-w64-x86-64
+#            - gcc-mingw-w64-x86-64
+#            - gfortran-mingw-w64-x86-64
+#      before_script: *common-before
+#      script:
+#        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+#      env:
+#        - TARGET_BOX=WIN64
+#        - BTYPE="BINARY=64 HOSTCC=gcc CC=x86_64-w64-mingw32-gcc FC=x86_64-w64-mingw32-gfortran"
+#
     # Build & test on Alpine Linux inside chroot, i.e. on system with musl libc.
     # These jobs needs sudo, so Travis runs them on VM-based infrastructure
     # which is slower than container-based infrastructure used for jobs
     # that don't require sudo.
-    - &test-alpine
-      os: linux
-      dist: trusty
-      sudo: true
-      language: minimal
-      before_install:
-        - "wget 'https://raw.githubusercontent.com/alpinelinux/alpine-chroot-install/v0.9.0/alpine-chroot-install' \
-          && echo 'e5dfbbdc0c4b3363b99334510976c86bfa6cb251  alpine-chroot-install' | sha1sum -c || exit 1"
-        - alpine() { /alpine/enter-chroot -u "$USER" "$@"; }
-      install:
-        - sudo sh alpine-chroot-install -p 'build-base gfortran perl linux-headers'
-      before_script: *common-before
-      script:
-        # XXX: Disable some warnings for now to avoid exceeding Travis limit for log size.
-        - alpine make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
-              CFLAGS="-Wno-misleading-indentation -Wno-sign-conversion -Wno-incompatible-pointer-types"
-        - alpine make -C test $COMMON_FLAGS $BTYPE
-        - alpine make -C ctest $COMMON_FLAGS $BTYPE
-        - alpine make -C utest $COMMON_FLAGS $BTYPE
-      env:
-        - TARGET_BOX=LINUX64_MUSL
-        - BTYPE="BINARY=64"
+ #   - &test-alpine
+ #     os: linux
+ #     dist: trusty
+ #     sudo: true
+ #     language: minimal
+ #     before_install:
+ #       - "wget 'https://raw.githubusercontent.com/alpinelinux/alpine-chroot-install/v0.9.0/alpine-chroot-install' \
+ #         && echo 'e5dfbbdc0c4b3363b99334510976c86bfa6cb251  alpine-chroot-install' | sha1sum -c || exit 1"
+ #       - alpine() { /alpine/enter-chroot -u "$USER" "$@"; }
+ #     install:
+ #       - sudo sh alpine-chroot-install -p 'build-base gfortran perl linux-headers'
+ #     before_script: *common-before
+ #     script:
+ #       # XXX: Disable some warnings for now to avoid exceeding Travis limit for log size.
+ #       - alpine make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+ #             CFLAGS="-Wno-misleading-indentation -Wno-sign-conversion -Wno-incompatible-pointer-types"
+ #       - alpine make -C test $COMMON_FLAGS $BTYPE
+ #       - alpine make -C ctest $COMMON_FLAGS $BTYPE
+ #       - alpine make -C utest $COMMON_FLAGS $BTYPE
+ #     env:
+ #       - TARGET_BOX=LINUX64_MUSL
+ #       - BTYPE="BINARY=64"
 
     # XXX: This job segfaults in TESTS OF THE COMPLEX LEVEL 3 BLAS,
     # but only on Travis CI, cannot reproduce it elsewhere.
@@ -171,98 +176,98 @@ matrix:
     #    - TARGET_BOX=LINUX64_MUSL
     #    - BTYPE="BINARY=64 USE_OPENMP=1"
 
-    - <<: *test-alpine
-      env:
-        - TARGET_BOX=LINUX64_MUSL
-        - BTYPE="BINARY=64 INTERFACE64=1"
+#    - <<: *test-alpine
+#      env:
+#        - TARGET_BOX=LINUX64_MUSL
+#        - BTYPE="BINARY=64 INTERFACE64=1"
+#
+#    # Build with the same flags as Alpine do in OpenBLAS package.
+#    - <<: *test-alpine
+#      env:
+#        - TARGET_BOX=LINUX64_MUSL
+#        - BTYPE="BINARY=64 NO_AFFINITY=1 USE_OPENMP=0 NO_LAPACK=0 TARGET=CORE2"
 
-    # Build with the same flags as Alpine do in OpenBLAS package.
-    - <<: *test-alpine
-      env:
-        - TARGET_BOX=LINUX64_MUSL
-        - BTYPE="BINARY=64 NO_AFFINITY=1 USE_OPENMP=0 NO_LAPACK=0 TARGET=CORE2"
+#    - &test-cmake
+#      os: linux
+#      compiler: clang
+#      addons:
+#        apt:
+#          packages:
+#            - gfortran
+#            - cmake
+#      dist: trusty
+#      sudo: true
+#      before_script:
+#        - COMMON_ARGS="-DTARGET=NEHALEM -DNUM_THREADS=32"
+#      script:
+#        - mkdir build
+#        - CONFIG=Release
+#        - cmake -Bbuild -H. $CMAKE_ARGS $COMMON_ARGS -DCMAKE_BUILD_TYPE=$CONFIG
+#        - cmake --build build --config $CONFIG -- -j2
+#      env:
+#        - CMAKE=1
+#    - <<: *test-cmake
+#      env:
+#        - CMAKE=1 CMAKE_ARGS="-DNOFORTRAN=1"
+#    - <<: *test-cmake
+#      compiler: gcc
+#      env:
+#        - CMAKE=1
 
-    - &test-cmake
-      os: linux
-      compiler: clang
-      addons:
-        apt:
-          packages:
-            - gfortran
-            - cmake
-      dist: trusty
-      sudo: true
-      before_script:
-        - COMMON_ARGS="-DTARGET=NEHALEM -DNUM_THREADS=32"
-      script:
-        - mkdir build
-        - CONFIG=Release
-        - cmake -Bbuild -H. $CMAKE_ARGS $COMMON_ARGS -DCMAKE_BUILD_TYPE=$CONFIG
-        - cmake --build build --config $CONFIG -- -j2
-      env:
-        - CMAKE=1
-    - <<: *test-cmake
-      env:
-        - CMAKE=1 CMAKE_ARGS="-DNOFORTRAN=1"
-    - <<: *test-cmake
-      compiler: gcc
-      env:
-        - CMAKE=1
-
-    - &test-macos
-      os: osx
-      osx_image: xcode11.5
-      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
-      script:
-        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
-      env:
-        - BTYPE="TARGET=NEHALEM BINARY=64 INTERFACE64=1 FC=gfortran-9"
-
-    - <<: *test-macos
-      osx_image: xcode12
-      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
-        - brew update
-      script:
-        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
-      env:
-        - BTYPE="TARGET=HASWELL USE_OPENMP=1 BINARY=64 INTERFACE64=1 CC=gcc-10 FC=gfortran-10"
-
-    - <<: *test-macos
-      osx_image: xcode12
-      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
-        - brew update
-      script:
-        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
-      env:
-        - BTYPE="TARGET=NEHALEM BINARY=64 INTERFACE64=1 FC=gfortran-10"      
+#    - &test-macos
+#      os: osx
+#      osx_image: xcode11.5
+#      before_script:
+#        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
+#      script:
+#        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+#      env:
+#        - BTYPE="TARGET=NEHALEM BINARY=64 INTERFACE64=1 FC=gfortran-9"
+#
+#    - <<: *test-macos
+#      osx_image: xcode12
+#      before_script:
+#        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
+#        - brew update
+#      script:
+#        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+#      env:
+#        - BTYPE="TARGET=HASWELL USE_OPENMP=1 BINARY=64 INTERFACE64=1 CC=gcc-10 FC=gfortran-10"
+#
+#    - <<: *test-macos
+#      osx_image: xcode12
+#      before_script:
+#        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
+#        - brew update
+#      script:
+#        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+#      env:
+#        - BTYPE="TARGET=NEHALEM BINARY=64 INTERFACE64=1 FC=gfortran-10"      
 
   #  - <<: *test-macos
   #    osx_image: xcode10
   #    env:
   #      - BTYPE="TARGET=NEHALEM BINARY=32 NOFORTRAN=1"
 
-    - <<: *test-macos
-      osx_image: xcode11.5
-      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
-        - brew update
-      env:
+#    - <<: *test-macos
+#      osx_image: xcode11.5
+#      before_script:
+#        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
+#        - brew update
+#      env:
 #        - CC="/Applications/Xcode-10.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
 #        - CFLAGS="-O2 -Wno-macro-redefined -isysroot /Applications/Xcode-10.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS12.1.sdk -arch arm64 -miphoneos-version-min=10.0"
-        - CC="/Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
-        - CFLAGS="-O2 -Wno-macro-redefined -isysroot /Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS13.5.sdk -arch arm64 -miphoneos-version-min=10.0"
-        - BTYPE="TARGET=ARMV8 BINARY=64 HOSTCC=clang NOFORTRAN=1"
-    - <<: *test-macos
-      osx_image: xcode11.5
-      env:
-#        - CC="/Applications/Xcode-10.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
-#        - CFLAGS="-O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode-10.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS12.1.sdk -arch armv7 -miphoneos-version-min=5.1"
-        - CC="/Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
-        - CFLAGS="-O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS13.5.sdk -arch armv7 -miphoneos-version-min=5.1"
-        - BTYPE="TARGET=ARMV7 HOSTCC=clang NOFORTRAN=1"
+#        - CC="/Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
+#        - CFLAGS="-O2 -Wno-macro-redefined -isysroot /Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS13.5.sdk -arch arm64 -miphoneos-version-min=10.0"
+#        - BTYPE="TARGET=ARMV8 BINARY=64 HOSTCC=clang NOFORTRAN=1"
+#    - <<: *test-macos
+#      osx_image: xcode11.5
+#      env:
+##        - CC="/Applications/Xcode-10.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
+##        - CFLAGS="-O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode-10.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS12.1.sdk -arch armv7 -miphoneos-version-min=5.1"
+#        - CC="/Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"
+#        - CFLAGS="-O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode-11.5.GM.Seed.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS13.5.sdk -arch armv7 -miphoneos-version-min=5.1"
+#        - BTYPE="TARGET=ARMV7 HOSTCC=clang NOFORTRAN=1"
 
     - &test-graviton2
       os: linux

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 17.dev)
+set(OpenBLAS_PATCH_VERSION 18)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 
 # Adhere to GNU filesystem layout conventions
@@ -132,7 +132,7 @@ endif ()
 
 if (BUILD_BFLOAT16)
   message(STATUS "Building Half Precision")
-  list(APPEND FLOAT_TYPES "BFLOAT16") # defines nothing
+  #  list(APPEND FLOAT_TYPES "BFLOAT16") # defines nothing
 endif ()
 
 if (NOT DEFINED CORE OR "${CORE}" STREQUAL "UNKNOWN")

Changelog.txt

@@ -1,4 +1,47 @@
 OpenBLAS ChangeLog
+====================================================================
+Version 0.3.18
+ 02-Oct-2021
+
+general:
+ - when the build-time number of preconfigured threads is exceeded
+   at runtime (typically by an external program calling BLAS functions
+   from a larger number of threads in parallel), OpenBLAS will now 
+   allocate an auxiliary control structure for up to 512 additional
+   threads instead of aborting
+ - added support for Loongson's LoongArch64 cpu architecture
+ - fixed building OpenBLAS with CMAKE and -DBUILD_BFLOAT16=ON
+ - added support for building OpenBLAS as a CMAKE subproject
+ - added support for building for Windows/ARM64 targets with clang
+ - improved support for building with the IBM xlf compiler
+ - imported Reference-LAPACK PR 625 (out-of-bounds reads in ?LARRV)
+ - imported Reference-LAPACK PR 597 for testsuite compatibility with
+   LLVM's libomp
+
+x86_64:
+ - added SkylakeX S/DGEMM kernels for small problem sizes (M*N*K<=1000000)
+ - added optimized SBGEMM for Intel Cooper Lake
+ - reinstated the performance patch for AVX512 SGEMV_T with a proper fix
+ - added a workaround for a gcc11 tree-vectorizer bug that caused spurious
+   failures in the test programs for complex BLAS3 when compiling at -O3
+   (the default for cmake "release" builds)
+ - added support for runtime cpu count detection under Haiku OS
+ - worked around a long-standing miscompilation issue of the Haswell DGEMV_T
+   kernel with gcc that could produce NaN output in some corner cases
+
+POWER:
+ - improved performance of DASUM on POWER10
+
+ARMV8:
+ - fixed crashes (use of reserved register x18) on Apple M1 under OSX
+ - fixed building with gcc releases earlier than 5.1
+
+MIPS:
+ - fixed building under BSD
+
+MIPS64:
+ - fixed building under BSD
+
 ====================================================================
 Version 0.3.17
  15-Jul-2021

Makefile

@@ -269,7 +269,7 @@ prof_lapack : lapack_prebuild
 lapack_prebuild :
 ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN)))
 	-@echo "FC          = $(FC)" > $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "FFLAGS      = $(LAPACK_FFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "override FFLAGS      = $(LAPACK_FFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "FFLAGS_DRV  = $(LAPACK_FFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "POPTS       = $(LAPACK_FPFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "FFLAGS_NOOPT       = -O0 $(LAPACK_NOOPT)" >> $(NETLIB_LAPACK_DIR)/make.inc

Makefile.power

@@ -12,9 +12,13 @@ endif
 ifeq ($(CORE), POWER10)
 ifneq ($(C_COMPILER), PGI)
 CCOMMON_OPT += -Ofast -mcpu=power10 -mtune=power10 -mvsx -fno-fast-math
+ifeq ($(F_COMPILER), IBM)
+FCOMMON_OPT += -O2 -qrecur -qnosave
+else
 FCOMMON_OPT += -O2 -frecursive -mcpu=power10 -mtune=power10  -fno-fast-math
 endif
 endif
+endif
 
 ifeq ($(CORE), POWER9)
 ifneq ($(C_COMPILER), PGI)
@@ -33,7 +37,11 @@ else
 CCOMMON_OPT += -fast -Mvect=simd -Mcache_align
 endif
 ifneq ($(F_COMPILER), PGI)
+ifeq ($(F_COMPILER), IBM)
+FCOMMON_OPT += -O2 -qrecur -qnosave
+else
 FCOMMON_OPT += -O2 -frecursive -fno-fast-math
+endif
 ifeq ($(C_COMPILER), GCC)
 ifneq ($(GCCVERSIONGT4), 1)
 $(warning your compiler is too old to fully support POWER9, getting a newer version of gcc is recommended)
@@ -57,7 +65,11 @@ CCOMMON_OPT += -fast -Mvect=simd -Mcache_align
 endif
 ifneq ($(F_COMPILER), PGI)
 ifeq ($(OSNAME), AIX)
+ifeq ($(F_COMPILER), IBM)
+FCOMMON_OPT += -O2 -qrecur -qnosave
+else
 FCOMMON_OPT += -O1 -frecursive -mcpu=power8 -mtune=power8  -fno-fast-math 
+endif
 else
 FCOMMON_OPT += -O2 -frecursive -mcpu=power8 -mtune=power8  -fno-fast-math 
 endif

Makefile.rule

@@ -3,7 +3,7 @@
 #
 
 # This library's version
-VERSION = 0.3.17.dev
+VERSION = 0.3.18
 
 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library

Makefile.system

@@ -33,6 +33,10 @@ else ifeq ($(ARCH), armv7)
 override ARCH=arm
 else ifeq ($(ARCH), aarch64)
 override ARCH=arm64
+else ifeq ($(ARCH), mipsel)
+override ARCH=mips
+else ifeq ($(ARCH), mips64el)
+override ARCH=mips64
 else ifeq ($(ARCH), zarch)
 override ARCH=zarch
 endif
@@ -244,6 +248,14 @@ else
 ONLY_CBLAS = 0
 endif
 
+#For small matrix optimization
+ifeq ($(ARCH), x86_64)
+SMALL_MATRIX_OPT = 1
+endif
+ifeq ($(SMALL_MATRIX_OPT), 1)
+CCOMMON_OPT += -DSMALL_MATRIX_OPT
+endif
+
 # This operation is expensive, so execution should be once.
 ifndef GOTOBLAS_MAKEFILE
 export GOTOBLAS_MAKEFILE = 1
@@ -856,7 +868,7 @@ BINARY_DEFINED = 1
 endif
 
 ifeq ($(ARCH), loongarch64)
-ifeq ($(CORE), LOONGSONG3R5)
+ifeq ($(CORE), LOONGSON3R5)
 CCOMMON_OPT += -march=loongarch64 -mabi=lp64
 FCOMMON_OPT += -march=loongarch64 -mabi=lp64
 endif

README.md

@@ -2,7 +2,7 @@
 
 [![Join the chat at https://gitter.im/xianyi/OpenBLAS](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/xianyi/OpenBLAS?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Travis CI: [![Build Status](https://travis-ci.org/xianyi/OpenBLAS.svg?branch=develop)](https://travis-ci.org/xianyi/OpenBLAS)
+Travis CI: [![Build Status](https://travis-ci.com/xianyi/OpenBLAS.svg?branch=develop)](https://travis-ci.com/xianyi/OpenBLAS)
 
 AppVeyor: [![Build status](https://ci.appveyor.com/api/projects/status/09sohd35n8nkkx64/branch/develop?svg=true)](https://ci.appveyor.com/project/xianyi/openblas/branch/develop)
 
@@ -128,6 +128,7 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th
 - **Intel Sandy Bridge**: Optimized Level-3 and Level-2 BLAS with AVX on x86-64.
 - **Intel Haswell**: Optimized Level-3 and Level-2 BLAS with AVX2 and FMA on x86-64.
 - **Intel Skylake-X**: Optimized Level-3 and Level-2 BLAS with AVX512 and FMA on x86-64.
+- **Intel Cooper Lake**: as Skylake-X with improved BFLOAT16 support.
 - **AMD Bobcat**: Used GotoBLAS2 Barcelona codes.
 - **AMD Bulldozer**: x86-64 ?GEMM FMA4 kernels. (Thanks to Werner Saar)
 - **AMD PILEDRIVER**: Uses Bulldozer codes with some optimizations.
@@ -153,6 +154,7 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th
 
 - **ARMv8**: Basic ARMV8 with small caches, optimized Level-3 and Level-2 BLAS
 - **Cortex-A53**: same as ARMV8 (different cpu specifications)
+- **Cortex-A55**: same as ARMV8 (different cpu specifications)
 - **Cortex A57**: Optimized Level-3 and Level-2 functions
 - **Cortex A72**: same as A57 ( different cpu specifications)
 - **Cortex A73**: same as A57 (different cpu specifications)
@@ -178,10 +180,11 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th
 
 #### RISC-V
 
-- **C910V**: Optimized Leve-3 BLAS (real) and Level-1,2 by RISC-V Vector extension 0.7.1.
+- **C910V**: Optimized Level-3 BLAS (real) and Level-1,2 by RISC-V Vector extension 0.7.1.
   ```sh
   make HOSTCC=gcc TARGET=C910V CC=riscv64-unknown-linux-gnu-gcc FC=riscv64-unknown-linux-gnu-gfortran
   ```
+  (also known to work on C906)
 
 ### Support for multiple targets in a single library
 

azure-pipelines.yml

@@ -19,7 +19,7 @@ jobs:
 # of gcc / glibc
 - job: manylinux1_gcc
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: |
       echo "FROM quay.io/pypa/manylinux1_x86_64
@@ -35,7 +35,7 @@ jobs:
     displayName: Run manylinux1 docker build
 - job: Intel_SDE_skx
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-latest'
   steps:
   - script: |
       # at the time of writing the available Azure Ubuntu vm image
@@ -83,6 +83,8 @@ jobs:
   - script: |
       brew update
       make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-10 FC=gfortran-10
+      make TARGET=CORE2 DYNAMIC_ARCH=1 USE_OPENMP=1 INTERFACE64=1 CC=gcc-10 FC=gfortran-10 PREFIX=../blasinst install
+      ls -lR ../blasinst
      
 - job: OSX_GCC_Nothreads
   pool:
@@ -104,6 +106,38 @@ jobs:
       brew install llvm libomp
       make TARGET=CORE2 USE_OPENMP=1 INTERFACE64=1 DYNAMIC_ARCH=1 CC=/usr/local/opt/llvm/bin/clang FC=gfortran-10
 
+- job: OSX_OpenMP_Clang_cmake
+  pool:
+     vmImage: 'macOS-10.15'
+  variables:
+     LD_LIBRARY_PATH: /usr/local/opt/llvm/lib
+     LIBRARY_PATH: /usr/local/opt/llvm/lib
+  steps:   
+  - script: |
+      brew update
+      brew install llvm libomp
+      mkdir build
+      cd build
+      cmake -DTARGET=CORE2 -DUSE_OPENMP=1 -DINTERFACE64=1 -DDYNAMIC_ARCH=1 -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang -DNOFORTRAN=1 -DNO_AVX512=1 ..
+      make
+      ctest
+      
+- job: OSX_OpenMP_Clang_gf_cmake
+  pool:
+     vmImage: 'macOS-10.15'
+  variables:
+     LD_LIBRARY_PATH: /usr/local/opt/llvm/lib
+     LIBRARY_PATH: /usr/local/opt/llvm/lib
+  steps:   
+  - script: |
+      brew update
+      brew install llvm libomp
+      mkdir build
+      cd build
+      cmake -DTARGET=CORE2 -DUSE_OPENMP=1 -DINTERFACE64=1 -DDYNAMIC_ARCH=1 -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang -DNO_AVX512=1 ..
+      make
+      ctest
+        
 - job: OSX_Ifort_Clang
   pool:
      vmImage: 'macOS-10.15'
@@ -146,14 +180,35 @@ jobs:
       brew install --cask android-ndk
       export ANDROID_NDK_HOME=/usr/local/share/android-ndk
       make TARGET=ARMV7 ONLY_CBLAS=1 CC=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi21-clang AR=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/darwin-x86_64/bin/arm-linux-androideabi-ar HOSTCC=gcc ARM_SOFTFP_ABI=1 -j4
- 
+
+- job: OSX_IOS_ARMV8
+  pool:
+     vmImage: 'macOS-10.15'
+  variables:
+     CC: /Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+     CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch arm64 -miphoneos-version-min=10.0
+  steps:
+  - script: |
+     make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1
+
+- job: OSX_IOS_ARMV7
+  pool:
+     vmImage: 'macOS-10.15'
+  variables:
+     CC: /Applications/Xcode_12.4.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+     CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_12.4.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS14.4.sdk -arch armv7 -miphoneos-version-min=5.1
+  steps:
+  - script: |
+     make TARGET=ARMV7 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1
+
 - job: ALPINE_MUSL
   pool:
      vmImage: 'ubuntu-latest'
   steps:
   - script: |
-        wget 'https://raw.githubusercontent.com/alpinelinux/alpine-chroot-install/v0.9.0/alpine-chroot-install' \
-          && echo 'e5dfbbdc0c4b3363b99334510976c86bfa6cb251  alpine-chroot-install' | sha1sum -c || exit 1
+        wget https://raw.githubusercontent.com/alpinelinux/alpine-chroot-install/v0.13.1/alpine-chroot-install \
+         && echo '7c7e3fa378e69aecc7f5f01bbc759e5f0a9d9b74  alpine-chroot-install' | sha1sum -c \
+         || exit 1
         alpine() { /alpine/enter-chroot -u "$USER" "$@"; }
         sudo sh alpine-chroot-install -p 'build-base gfortran perl linux-headers sudo'
         alpine make DYNAMIC_ARCH=1 BINARY=64

cblas.h

@@ -400,6 +400,8 @@ void   cblas_dbf16tod(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPE
 float  cblas_sbdot(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 *y, OPENBLAS_CONST blasint incy);
 void   cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order,  OPENBLAS_CONST enum CBLAS_TRANSPOSE trans,  OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST float beta, float *y, OPENBLAS_CONST blasint incy);
 
+void   cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
+		    OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc);
 #ifdef __cplusplus
 }
 #endif  /* __cplusplus */

cmake/arch.cmake

@@ -113,6 +113,10 @@ if (MIPS64)
   set(NO_BINARY_MODE 1)
 endif ()
 
+if (LOONGARCH64)
+  set(NO_BINARY_MODE 1)
+endif ()
+
 if (${ARCH} STREQUAL "alpha")
   set(NO_BINARY_MODE 1)
   set(BINARY_DEFINED 1)

cmake/cc.cmake

@@ -29,6 +29,15 @@ if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LS
       set(FCOMMON_OPT "${FCOMMON_OPT} -march=mips64")
     endif ()
 
+    if (LOONGARCH64)
+      if (BINARY64)
+        set(CCOMMON_OPT "${CCOMMON_OPT} -mabi=lp64")
+      else ()
+        set(CCOMMON_OPT "${CCOMMON_OPT} -mabi=lp32")
+      endif ()
+      set(BINARY_DEFINED 1)
+    endif ()
+
     if (CMAKE_SYSTEM_NAME STREQUAL "AIX")
       set(BINARY_DEFINED 1)
     endif ()

cmake/fc.cmake

@@ -61,6 +61,13 @@ if (${F_COMPILER} STREQUAL "GFORTRAN")
         set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=n32")
       endif ()
     endif ()
+    if (LOONGARCH64)
+      if (BINARY64)
+        set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=lp64")
+      else ()
+        set(FCOMMON_OPT "${FCOMMON_OPT} -mabi=lp32")
+      endif ()
+    endif ()
   else ()
     if (BINARY64)
       set(FCOMMON_OPT "${FCOMMON_OPT} -m64")
@@ -97,7 +104,7 @@ endif ()
 
 if (${F_COMPILER} STREQUAL "IBM")
   set(CCOMMON_OPT "${CCOMMON_OPT} -DF_INTERFACE_IBM")
-  # FCOMMON_OPT	+= -qarch=440
+  set(FCOMMON_OPT "${FCOMMON_OPT} -qrecur")
   if (BINARY64)
     set(FCOMMON_OPT "${FCOMMON_OPT} -q64")
     if (INTERFACE64)

cmake/kernel.cmake

@@ -134,6 +134,8 @@ if (BUILD_BFLOAT16)
   set(SHSWAPKERNEL ../arm/swap.c)
   set(TOBF16KERNEL ../x86_64/tobf16.c)
   set(BF16TOKERNEL ../x86_64/bf16to.c)
+  set(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
+  set(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
 endif ()
 endmacro ()
 

cmake/system.cmake

@@ -258,6 +258,13 @@ if (NEED_PIC)
   endif()
 endif ()
 
+if (X86_64)
+  set(SMALL_MATRIX_OPT TRUE)
+endif ()
+if (SMALL_MATRIX_OPT)
+  set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT")
+endif ()
+
 if (DYNAMIC_ARCH)
   if (X86 OR X86_64 OR ARM64 OR PPC)
     set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH")
@@ -462,6 +469,9 @@ endif()
 if (BUILD_COMPLEX16)
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBUILD_COMPLEX16")
 endif()
+if (BUILD_BFLOAT16)
+       set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBUILD_BFLOAT16")
+endif()
 if(NOT MSVC)
 set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${CCOMMON_OPT}")
 endif()

cmake/system_check.cmake

@@ -38,6 +38,8 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc.*|power.*|Power.*")
   set(PPC 1)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "mips64.*")
   set(MIPS64 1)
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64.*")
+  set(LOONGARCH64 1)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
   if (NOT BINARY)
     if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
@@ -95,7 +97,7 @@ else()
 endif ()
 
 if (NOT BINARY)
-  if (X86_64 OR ARM64 OR PPC OR MIPS64)
+  if (X86_64 OR ARM64 OR PPC OR MIPS64 OR LOONGARCH64)
     set(BINARY 64)
   else ()
     set(BINARY 32)

cmake/utils.cmake

@@ -157,31 +157,31 @@ endfunction ()
 #                               STRING - compiles only the given type (e.g. DOUBLE)
 function(GenerateNamedObjects sources_in)
 
-  if (DEFINED ARGV1)
+  if (${ARGC} GREATER 1)
     set(defines_in ${ARGV1})
   endif ()
 
-  if (DEFINED ARGV2 AND NOT "${ARGV2}" STREQUAL "")
+  if (${ARGC} GREATER 2 AND NOT "${ARGV2}" STREQUAL "")
     set(name_in ${ARGV2})
     # strip off extension for kernel files that pass in the object name.
     get_filename_component(name_in ${name_in} NAME_WE)
   endif ()
 
-  if (DEFINED ARGV3)
+  if (${ARGC} GREATER 3)
     set(use_cblas ${ARGV3})
   else ()
     set(use_cblas false)
   endif ()
 
-  if (DEFINED ARGV4)
+  if (${ARGC} GREATER 4)
     set(replace_last_with ${ARGV4})
   endif ()
 
-  if (DEFINED ARGV5)
+  if (${ARGC} GREATER 5)
     set(append_with ${ARGV5})
   endif ()
 
-  if (DEFINED ARGV6)
+  if (${ARGC} GREATER 6)
     set(no_float_type ${ARGV6})
   else ()
     set(no_float_type false)
@@ -196,7 +196,7 @@ function(GenerateNamedObjects sources_in)
   set(real_only false)
   set(complex_only false)
   set(mangle_complex_sources false)
-  if (DEFINED ARGV7 AND NOT "${ARGV7}" STREQUAL "")
+  if (${ARGC} GREATER 7 AND NOT "${ARGV7}" STREQUAL "")
     if (${ARGV7} EQUAL 1)
       set(real_only true)
     elseif (${ARGV7} EQUAL 2)
@@ -342,17 +342,17 @@ endfunction ()
 function(GenerateCombinationObjects sources_in defines_in absent_codes_in all_defines_in replace_scheme)
 
   set(alternate_name_in "")
-  if (DEFINED ARGV5)
+  if (${ARGC} GREATER 5)
     set(alternate_name_in ${ARGV5})
   endif ()
 
   set(no_float_type false)
-  if (DEFINED ARGV6)
+  if (${ARGC} GREATER 6)
     set(no_float_type ${ARGV6})
   endif ()
 
   set(complex_filename_scheme "")
-  if (DEFINED ARGV7)
+  if (${ARGC} GREATER 7)
     set(complex_filename_scheme ${ARGV7})
   endif ()
 

common_arm64.h

@@ -120,7 +120,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 	.text ;
 	.p2align 2 ;
 	.global	REALNAME ;
-#ifndef __APPLE__
+#if !defined(__APPLE__) && !defined(_WIN32)
 	.type	REALNAME, %function ;
 #endif
 REALNAME:

common_c.h

@@ -232,6 +232,8 @@
 
 #define CGEADD_K                cgeadd_k 
 
+#define CGEMM_SMALL_MATRIX_PERMIT	cgemm_small_matrix_permit
+
 #else
 
 #define	CAMAX_K			gotoblas -> camax_k
@@ -426,8 +428,51 @@
 
 #define CGEADD_K                gotoblas -> cgeadd_k 
 
+#define CGEMM_SMALL_MATRIX_PERMIT	gotoblas -> cgemm_small_matrix_permit
+
 #endif
 
+#define CGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(cgemm_small_kernel_nn)
+#define CGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(cgemm_small_kernel_nt)
+#define CGEMM_SMALL_KERNEL_NR		FUNC_OFFSET(cgemm_small_kernel_nr)
+#define CGEMM_SMALL_KERNEL_NC		FUNC_OFFSET(cgemm_small_kernel_nc)
+
+#define CGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(cgemm_small_kernel_tn)
+#define CGEMM_SMALL_KERNEL_TT		FUNC_OFFSET(cgemm_small_kernel_tt)
+#define CGEMM_SMALL_KERNEL_TR		FUNC_OFFSET(cgemm_small_kernel_tr)
+#define CGEMM_SMALL_KERNEL_TC		FUNC_OFFSET(cgemm_small_kernel_tc)
+
+#define CGEMM_SMALL_KERNEL_RN		FUNC_OFFSET(cgemm_small_kernel_rn)
+#define CGEMM_SMALL_KERNEL_RT		FUNC_OFFSET(cgemm_small_kernel_rt)
+#define CGEMM_SMALL_KERNEL_RR		FUNC_OFFSET(cgemm_small_kernel_rr)
+#define CGEMM_SMALL_KERNEL_RC		FUNC_OFFSET(cgemm_small_kernel_rc)
+
+#define CGEMM_SMALL_KERNEL_CN		FUNC_OFFSET(cgemm_small_kernel_cn)
+#define CGEMM_SMALL_KERNEL_CT		FUNC_OFFSET(cgemm_small_kernel_ct)
+#define CGEMM_SMALL_KERNEL_CR		FUNC_OFFSET(cgemm_small_kernel_cr)
+#define CGEMM_SMALL_KERNEL_CC		FUNC_OFFSET(cgemm_small_kernel_cc)
+
+#define CGEMM_SMALL_KERNEL_B0_NN	FUNC_OFFSET(cgemm_small_kernel_b0_nn)
+#define CGEMM_SMALL_KERNEL_B0_NT	FUNC_OFFSET(cgemm_small_kernel_b0_nt)
+#define CGEMM_SMALL_KERNEL_B0_NR	FUNC_OFFSET(cgemm_small_kernel_b0_nr)
+#define CGEMM_SMALL_KERNEL_B0_NC	FUNC_OFFSET(cgemm_small_kernel_b0_nc)
+
+#define CGEMM_SMALL_KERNEL_B0_TN	FUNC_OFFSET(cgemm_small_kernel_b0_tn)
+#define CGEMM_SMALL_KERNEL_B0_TT	FUNC_OFFSET(cgemm_small_kernel_b0_tt)
+#define CGEMM_SMALL_KERNEL_B0_TR	FUNC_OFFSET(cgemm_small_kernel_b0_tr)
+#define CGEMM_SMALL_KERNEL_B0_TC	FUNC_OFFSET(cgemm_small_kernel_b0_tc)
+
+#define CGEMM_SMALL_KERNEL_B0_RN	FUNC_OFFSET(cgemm_small_kernel_b0_rn)
+#define CGEMM_SMALL_KERNEL_B0_RT	FUNC_OFFSET(cgemm_small_kernel_b0_rt)
+#define CGEMM_SMALL_KERNEL_B0_RR	FUNC_OFFSET(cgemm_small_kernel_b0_rr)
+#define CGEMM_SMALL_KERNEL_B0_RC	FUNC_OFFSET(cgemm_small_kernel_b0_rc)
+
+#define CGEMM_SMALL_KERNEL_B0_CN	FUNC_OFFSET(cgemm_small_kernel_b0_cn)
+#define CGEMM_SMALL_KERNEL_B0_CT	FUNC_OFFSET(cgemm_small_kernel_b0_ct)
+#define CGEMM_SMALL_KERNEL_B0_CR	FUNC_OFFSET(cgemm_small_kernel_b0_cr)
+#define CGEMM_SMALL_KERNEL_B0_CC	FUNC_OFFSET(cgemm_small_kernel_b0_cc)
+
+
 #define	CGEMM_NN		cgemm_nn
 #define	CGEMM_CN		cgemm_cn
 #define	CGEMM_TN		cgemm_tn

common_d.h

@@ -157,6 +157,8 @@
 #define DIMATCOPY_K_RT      dimatcopy_k_rt
 #define DGEADD_K                dgeadd_k 
 
+#define DGEMM_SMALL_MATRIX_PERMIT	dgemm_small_matrix_permit
+
 #else
 
 #define	DAMAX_K			gotoblas -> damax_k
@@ -281,8 +283,21 @@
 
 #define DGEADD_K                gotoblas -> dgeadd_k 
 
+#define DGEMM_SMALL_MATRIX_PERMIT	gotoblas -> dgemm_small_matrix_permit
+
 #endif
 
+#define DGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(dgemm_small_kernel_nn)
+#define DGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(dgemm_small_kernel_nt)
+#define DGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(dgemm_small_kernel_tn)
+#define DGEMM_SMALL_KERNEL_TT		FUNC_OFFSET(dgemm_small_kernel_tt)
+
+#define DGEMM_SMALL_KERNEL_B0_NN	FUNC_OFFSET(dgemm_small_kernel_b0_nn)
+#define DGEMM_SMALL_KERNEL_B0_NT	FUNC_OFFSET(dgemm_small_kernel_b0_nt)
+#define DGEMM_SMALL_KERNEL_B0_TN	FUNC_OFFSET(dgemm_small_kernel_b0_tn)
+#define DGEMM_SMALL_KERNEL_B0_TT	FUNC_OFFSET(dgemm_small_kernel_b0_tt)
+
+
 #define	DGEMM_NN		dgemm_nn
 #define	DGEMM_CN		dgemm_tn
 #define	DGEMM_TN		dgemm_tn

common_level3.h

@@ -515,6 +515,129 @@ int qgemm_kernel(BLASLONG, BLASLONG, BLASLONG, xidouble *, xidouble *, xidouble
 int qgemm_kernel(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xdouble *, BLASLONG);
 #endif
 
+#ifdef SMALL_MATRIX_OPT
+int sbgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta);
+
+int sbgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+
+int sgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta);
+
+int sgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+int sgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+
+int dgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double beta);
+
+int dgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+int dgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+int dgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+int dgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+
+int sbgemm_small_kernel_b0_nn(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sbgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+
+int sgemm_small_kernel_b0_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+int sgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+
+int dgemm_small_kernel_b0_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+int dgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+int dgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+int dgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+
+int cgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha0, float alpha1, float beta0, float beta1);
+
+int cgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_nc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+	
+int cgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_tr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_tc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+int cgemm_small_kernel_rn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_rt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_rr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_rc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+int cgemm_small_kernel_cn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_ct(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_cr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+int cgemm_small_kernel_cc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+int zgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha0, double alpha1, double beta0, double beta1);
+
+int zgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_nc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+	
+int zgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_tr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_tc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+int zgemm_small_kernel_rn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_rt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_rr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_rc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+int zgemm_small_kernel_cn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_ct(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_cr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+int zgemm_small_kernel_cc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+int cgemm_small_kernel_b0_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);	
+int cgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_nr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_nc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+	
+int cgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_tr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_tc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+int cgemm_small_kernel_b0_rn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_rt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_rr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_rc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+int cgemm_small_kernel_b0_cn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_ct(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_cr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+int cgemm_small_kernel_b0_cc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+int zgemm_small_kernel_b0_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);	
+int zgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_nr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_nc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+	
+int zgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_tr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_tc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+int zgemm_small_kernel_b0_rn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_rt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_rr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_rc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+int zgemm_small_kernel_b0_cn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_ct(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_cr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+int zgemm_small_kernel_b0_cc(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+#endif
+
 int cgemm_kernel_n(BLASLONG, BLASLONG, BLASLONG, float,  float,  float  *, float  *, float  *, BLASLONG);
 int cgemm_kernel_l(BLASLONG, BLASLONG, BLASLONG, float,  float,  float  *, float  *, float  *, BLASLONG);
 int cgemm_kernel_r(BLASLONG, BLASLONG, BLASLONG, float,  float,  float  *, float  *, float  *, BLASLONG);

common_loongarch64.h

@@ -186,7 +186,7 @@ REALNAME: ;\
 
 #define BUFFER_SIZE     ( 32 << 20)
 
-#define PAGESIZE        (16UL << 1)
+#define PAGESIZE        (16UL << 10)
 #define FIXED_PAGESIZE  (16UL << 10)
 #define HUGE_PAGESIZE   ( 2 << 20)
 

common_macro.h

@@ -644,6 +644,17 @@
 
 #define GEADD_K                 DGEADD_K
 
+#define GEMM_SMALL_MATRIX_PERMIT	DGEMM_SMALL_MATRIX_PERMIT
+
+#define GEMM_SMALL_KERNEL_NN    DGEMM_SMALL_KERNEL_NN
+#define GEMM_SMALL_KERNEL_NT    DGEMM_SMALL_KERNEL_NT
+#define GEMM_SMALL_KERNEL_TN    DGEMM_SMALL_KERNEL_TN
+#define GEMM_SMALL_KERNEL_TT    DGEMM_SMALL_KERNEL_TT
+#define GEMM_SMALL_KERNEL_B0_NN    DGEMM_SMALL_KERNEL_B0_NN
+#define GEMM_SMALL_KERNEL_B0_NT    DGEMM_SMALL_KERNEL_B0_NT
+#define GEMM_SMALL_KERNEL_B0_TN    DGEMM_SMALL_KERNEL_B0_TN
+#define GEMM_SMALL_KERNEL_B0_TT    DGEMM_SMALL_KERNEL_B0_TT
+
 #elif defined(BFLOAT16)
 
 #define D_TO_BF16_K     SBDTOBF16_K
@@ -931,6 +942,18 @@
 
 #define GEADD_K 		SGEADD_K
 
+#define GEMM_SMALL_MATRIX_PERMIT	SBGEMM_SMALL_MATRIX_PERMIT
+
+#define GEMM_SMALL_KERNEL_NN    SBGEMM_SMALL_KERNEL_NN
+#define GEMM_SMALL_KERNEL_NT    SBGEMM_SMALL_KERNEL_NT
+#define GEMM_SMALL_KERNEL_TN    SBGEMM_SMALL_KERNEL_TN
+#define GEMM_SMALL_KERNEL_TT    SBGEMM_SMALL_KERNEL_TT
+
+#define GEMM_SMALL_KERNEL_B0_NN    SBGEMM_SMALL_KERNEL_B0_NN
+#define GEMM_SMALL_KERNEL_B0_NT    SBGEMM_SMALL_KERNEL_B0_NT
+#define GEMM_SMALL_KERNEL_B0_TN    SBGEMM_SMALL_KERNEL_B0_TN
+#define GEMM_SMALL_KERNEL_B0_TT    SBGEMM_SMALL_KERNEL_B0_TT
+
 #endif
 
 #else
@@ -1236,6 +1259,19 @@
 #define IMATCOPY_K_RT		SIMATCOPY_K_RT
 
 #define GEADD_K 		SGEADD_K
+
+#define GEMM_SMALL_MATRIX_PERMIT	SGEMM_SMALL_MATRIX_PERMIT
+
+#define GEMM_SMALL_KERNEL_NN    SGEMM_SMALL_KERNEL_NN
+#define GEMM_SMALL_KERNEL_NT    SGEMM_SMALL_KERNEL_NT
+#define GEMM_SMALL_KERNEL_TN    SGEMM_SMALL_KERNEL_TN
+#define GEMM_SMALL_KERNEL_TT    SGEMM_SMALL_KERNEL_TT
+
+#define GEMM_SMALL_KERNEL_B0_NN    SGEMM_SMALL_KERNEL_B0_NN
+#define GEMM_SMALL_KERNEL_B0_NT    SGEMM_SMALL_KERNEL_B0_NT
+#define GEMM_SMALL_KERNEL_B0_TN    SGEMM_SMALL_KERNEL_B0_TN
+#define GEMM_SMALL_KERNEL_B0_TT    SGEMM_SMALL_KERNEL_B0_TT
+
 #endif
 #else
 #ifdef XDOUBLE
@@ -2063,6 +2099,48 @@
 
 #define GEADD_K                 ZGEADD_K
 
+#define GEMM_SMALL_MATRIX_PERMIT	ZGEMM_SMALL_MATRIX_PERMIT
+
+#define GEMM_SMALL_KERNEL_NN    ZGEMM_SMALL_KERNEL_NN
+#define GEMM_SMALL_KERNEL_NT    ZGEMM_SMALL_KERNEL_NT
+#define GEMM_SMALL_KERNEL_NR    ZGEMM_SMALL_KERNEL_NR
+#define GEMM_SMALL_KERNEL_NC    ZGEMM_SMALL_KERNEL_NC
+
+#define GEMM_SMALL_KERNEL_TN    ZGEMM_SMALL_KERNEL_TN
+#define GEMM_SMALL_KERNEL_TT    ZGEMM_SMALL_KERNEL_TT
+#define GEMM_SMALL_KERNEL_TR    ZGEMM_SMALL_KERNEL_TR
+#define GEMM_SMALL_KERNEL_TC    ZGEMM_SMALL_KERNEL_TC
+
+#define GEMM_SMALL_KERNEL_RN    ZGEMM_SMALL_KERNEL_RN
+#define GEMM_SMALL_KERNEL_RT    ZGEMM_SMALL_KERNEL_RT
+#define GEMM_SMALL_KERNEL_RR    ZGEMM_SMALL_KERNEL_RR
+#define GEMM_SMALL_KERNEL_RC    ZGEMM_SMALL_KERNEL_RC
+
+#define GEMM_SMALL_KERNEL_CN    ZGEMM_SMALL_KERNEL_CN
+#define GEMM_SMALL_KERNEL_CT    ZGEMM_SMALL_KERNEL_CT
+#define GEMM_SMALL_KERNEL_CR    ZGEMM_SMALL_KERNEL_CR
+#define GEMM_SMALL_KERNEL_CC    ZGEMM_SMALL_KERNEL_CC
+
+#define GEMM_SMALL_KERNEL_B0_NN    ZGEMM_SMALL_KERNEL_B0_NN
+#define GEMM_SMALL_KERNEL_B0_NT    ZGEMM_SMALL_KERNEL_B0_NT
+#define GEMM_SMALL_KERNEL_B0_NR    ZGEMM_SMALL_KERNEL_B0_NR
+#define GEMM_SMALL_KERNEL_B0_NC    ZGEMM_SMALL_KERNEL_B0_NC
+
+#define GEMM_SMALL_KERNEL_B0_TN    ZGEMM_SMALL_KERNEL_B0_TN
+#define GEMM_SMALL_KERNEL_B0_TT    ZGEMM_SMALL_KERNEL_B0_TT
+#define GEMM_SMALL_KERNEL_B0_TR    ZGEMM_SMALL_KERNEL_B0_TR
+#define GEMM_SMALL_KERNEL_B0_TC    ZGEMM_SMALL_KERNEL_B0_TC
+
+#define GEMM_SMALL_KERNEL_B0_RN    ZGEMM_SMALL_KERNEL_B0_RN
+#define GEMM_SMALL_KERNEL_B0_RT    ZGEMM_SMALL_KERNEL_B0_RT
+#define GEMM_SMALL_KERNEL_B0_RR    ZGEMM_SMALL_KERNEL_B0_RR
+#define GEMM_SMALL_KERNEL_B0_RC    ZGEMM_SMALL_KERNEL_B0_RC
+
+#define GEMM_SMALL_KERNEL_B0_CN    ZGEMM_SMALL_KERNEL_B0_CN
+#define GEMM_SMALL_KERNEL_B0_CT    ZGEMM_SMALL_KERNEL_B0_CT
+#define GEMM_SMALL_KERNEL_B0_CR    ZGEMM_SMALL_KERNEL_B0_CR
+#define GEMM_SMALL_KERNEL_B0_CC    ZGEMM_SMALL_KERNEL_B0_CC
+
 #else
 
 #define	AMAX_K			CAMAX_K
@@ -2486,6 +2564,48 @@
 
 #define GEADD_K                 CGEADD_K
 
+#define GEMM_SMALL_MATRIX_PERMIT	CGEMM_SMALL_MATRIX_PERMIT
+
+#define GEMM_SMALL_KERNEL_NN    CGEMM_SMALL_KERNEL_NN
+#define GEMM_SMALL_KERNEL_NT    CGEMM_SMALL_KERNEL_NT
+#define GEMM_SMALL_KERNEL_NR    CGEMM_SMALL_KERNEL_NR
+#define GEMM_SMALL_KERNEL_NC    CGEMM_SMALL_KERNEL_NC
+
+#define GEMM_SMALL_KERNEL_TN    CGEMM_SMALL_KERNEL_TN
+#define GEMM_SMALL_KERNEL_TT    CGEMM_SMALL_KERNEL_TT
+#define GEMM_SMALL_KERNEL_TR    CGEMM_SMALL_KERNEL_TR
+#define GEMM_SMALL_KERNEL_TC    CGEMM_SMALL_KERNEL_TC
+
+#define GEMM_SMALL_KERNEL_RN    CGEMM_SMALL_KERNEL_RN
+#define GEMM_SMALL_KERNEL_RT    CGEMM_SMALL_KERNEL_RT
+#define GEMM_SMALL_KERNEL_RR    CGEMM_SMALL_KERNEL_RR
+#define GEMM_SMALL_KERNEL_RC    CGEMM_SMALL_KERNEL_RC
+
+#define GEMM_SMALL_KERNEL_CN    CGEMM_SMALL_KERNEL_CN
+#define GEMM_SMALL_KERNEL_CT    CGEMM_SMALL_KERNEL_CT
+#define GEMM_SMALL_KERNEL_CR    CGEMM_SMALL_KERNEL_CR
+#define GEMM_SMALL_KERNEL_CC    CGEMM_SMALL_KERNEL_CC
+
+#define GEMM_SMALL_KERNEL_B0_NN    CGEMM_SMALL_KERNEL_B0_NN
+#define GEMM_SMALL_KERNEL_B0_NT    CGEMM_SMALL_KERNEL_B0_NT
+#define GEMM_SMALL_KERNEL_B0_NR    CGEMM_SMALL_KERNEL_B0_NR
+#define GEMM_SMALL_KERNEL_B0_NC    CGEMM_SMALL_KERNEL_B0_NC
+
+#define GEMM_SMALL_KERNEL_B0_TN    CGEMM_SMALL_KERNEL_B0_TN
+#define GEMM_SMALL_KERNEL_B0_TT    CGEMM_SMALL_KERNEL_B0_TT
+#define GEMM_SMALL_KERNEL_B0_TR    CGEMM_SMALL_KERNEL_B0_TR
+#define GEMM_SMALL_KERNEL_B0_TC    CGEMM_SMALL_KERNEL_B0_TC
+
+#define GEMM_SMALL_KERNEL_B0_RN    CGEMM_SMALL_KERNEL_B0_RN
+#define GEMM_SMALL_KERNEL_B0_RT    CGEMM_SMALL_KERNEL_B0_RT
+#define GEMM_SMALL_KERNEL_B0_RR    CGEMM_SMALL_KERNEL_B0_RR
+#define GEMM_SMALL_KERNEL_B0_RC    CGEMM_SMALL_KERNEL_B0_RC
+
+#define GEMM_SMALL_KERNEL_B0_CN    CGEMM_SMALL_KERNEL_B0_CN
+#define GEMM_SMALL_KERNEL_B0_CT    CGEMM_SMALL_KERNEL_B0_CT
+#define GEMM_SMALL_KERNEL_B0_CR    CGEMM_SMALL_KERNEL_B0_CR
+#define GEMM_SMALL_KERNEL_B0_CC    CGEMM_SMALL_KERNEL_B0_CC
+
 #endif
 #endif
 

common_param.h

@@ -145,6 +145,19 @@ BLASLONG (*isbmin_k) (BLASLONG, float *, BLASLONG);
   int	 (*sbneg_tcopy)   (BLASLONG, BLASLONG, float *, BLASLONG, float *);
   int    (*sblaswp_ncopy) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG, blasint *, float *);
 
+#ifdef SMALL_MATRIX_OPT
+  int    (*sbgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta);
+
+  int    (*sbgemm_small_kernel_nn    )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_nt    )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_tn    )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_tt    )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+
+  int    (*sbgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sbgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, bfloat16 * A, BLASLONG lda, float alpha, bfloat16 * B, BLASLONG ldb, float * C, BLASLONG ldc);
+#endif
 #endif
 
 #if defined(BUILD_SINGLE) || defined(BUILD_COMPLEX)
@@ -207,6 +220,20 @@ BLASLONG (*ismin_k) (BLASLONG, float *, BLASLONG);
   int    (*sgemm_otcopy   )(BLASLONG, BLASLONG, float *, BLASLONG, float *);
 #endif
 #ifdef BUILD_SINGLE  
+#ifdef SMALL_MATRIX_OPT
+  int    (*sgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta);
+
+  int    (*sgemm_small_kernel_nn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_nt    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_tn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_tt    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc);
+
+  int    (*sgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+  int    (*sgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float * C, BLASLONG ldc);
+#endif
+
   int    (*strsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG);
   int    (*strsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG);
   int    (*strsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG, BLASLONG);
@@ -314,6 +341,19 @@ BLASLONG (*idmin_k) (BLASLONG, double *, BLASLONG);
   int    (*dgemm_otcopy   )(BLASLONG, BLASLONG, double *, BLASLONG, double *);
 #endif
 #ifdef BUILD_DOUBLE
+#ifdef SMALL_MATRIX_OPT
+  int    (*dgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double beta);
+
+  int    (*dgemm_small_kernel_nn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_nt    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_tn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_tt    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc);
+
+  int    (*dgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+  int    (*dgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc);
+#endif
   int    (*dtrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG);
   int    (*dtrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG);
   int    (*dtrsm_kernel_RN)(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG, BLASLONG);
@@ -513,6 +553,50 @@ BLASLONG (*icamin_k)(BLASLONG, float *, BLASLONG);
   int    (*cgemm_oncopy   )(BLASLONG, BLASLONG, float *, BLASLONG, float *);
   int    (*cgemm_otcopy   )(BLASLONG, BLASLONG, float *, BLASLONG, float *);
 
+#ifdef SMALL_MATRIX_OPT
+  int    (*cgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha0, float alpha1, float beta0, float beta1);
+
+  int    (*cgemm_small_kernel_nn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_nt    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_nr    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_nc    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_tn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_tt    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_tr    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_tc    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_rn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_rt    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_rr    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_rc    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_cn    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_ct    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_cr    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_cc    )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+
+  int    (*cgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+  int    (*cgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb,  float * C, BLASLONG ldc);
+#endif
+
   int    (*ctrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
   int    (*ctrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
   int    (*ctrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, float, float, float *, float *, float *, BLASLONG, BLASLONG);
@@ -679,6 +763,50 @@ BLASLONG (*izamin_k)(BLASLONG, double *, BLASLONG);
   int    (*zgemm_oncopy   )(BLASLONG, BLASLONG, double *, BLASLONG, double *);
   int    (*zgemm_otcopy   )(BLASLONG, BLASLONG, double *, BLASLONG, double *);
 
+#ifdef SMALL_MATRIX_OPT
+  int    (*zgemm_small_matrix_permit)(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha0, double alpha1, double beta0, double beta1);
+
+  int    (*zgemm_small_kernel_nn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_nt    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_nr    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_nc    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_tn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_tt    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_tr    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_tc    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_rn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_rt    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_rr    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_rc    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_cn    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_ct    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_cr    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_cc    )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_b0_nn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_nt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_nr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_nc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_b0_tn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_tt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_tr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_tc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_b0_rn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_rt )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_rr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_rc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+
+  int    (*zgemm_small_kernel_b0_cn )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_ct )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_cr )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+  int    (*zgemm_small_kernel_b0_cc )(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb,  double * C, BLASLONG ldc);
+#endif
+
   int    (*ztrsm_kernel_LN)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
   int    (*ztrsm_kernel_LT)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
   int    (*ztrsm_kernel_LR)(BLASLONG, BLASLONG, BLASLONG, double, double, double *, double *, double *, BLASLONG, BLASLONG);
@@ -1069,6 +1197,8 @@ BLASLONG (*ixamin_k)(BLASLONG, xdouble *, BLASLONG);
 
 extern gotoblas_t *gotoblas;
 
+#define FUNC_OFFSET(func)	(size_t)(&((gotoblas_t *)NULL)->func)
+
 #define DTB_ENTRIES  gotoblas -> dtb_entries
 #define GEMM_OFFSET_A	gotoblas -> offsetA
 #define GEMM_OFFSET_B	gotoblas -> offsetB
@@ -1174,6 +1304,8 @@ extern gotoblas_t *gotoblas;
 
 #else
 
+#define FUNC_OFFSET(func)	(size_t)(func)
+
 #define DTB_ENTRIES  DTB_DEFAULT_ENTRIES
 
 #define GEMM_OFFSET_A	GEMM_DEFAULT_OFFSET_A

common_s.h

@@ -164,6 +164,8 @@
 
 #define SGEADD_K                sgeadd_k 
 
+#define SGEMM_SMALL_MATRIX_PERMIT	sgemm_small_matrix_permit
+
 #else
 
 #define	SAMAX_K			gotoblas -> samax_k
@@ -299,8 +301,21 @@
 
 #define SGEADD_K                gotoblas -> sgeadd_k 
 
+#define SGEMM_SMALL_MATRIX_PERMIT	gotoblas -> sgemm_small_matrix_permit
+
 #endif
 
+#define SGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(sgemm_small_kernel_nn)
+#define SGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(sgemm_small_kernel_nt)
+#define SGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(sgemm_small_kernel_tn)
+#define SGEMM_SMALL_KERNEL_TT		FUNC_OFFSET(sgemm_small_kernel_tt)
+
+#define SGEMM_SMALL_KERNEL_B0_NN	FUNC_OFFSET(sgemm_small_kernel_b0_nn)
+#define SGEMM_SMALL_KERNEL_B0_NT	FUNC_OFFSET(sgemm_small_kernel_b0_nt)
+#define SGEMM_SMALL_KERNEL_B0_TN	FUNC_OFFSET(sgemm_small_kernel_b0_tn)
+#define SGEMM_SMALL_KERNEL_B0_TT	FUNC_OFFSET(sgemm_small_kernel_b0_tt)
+
+
 #define	SGEMM_NN		sgemm_nn
 #define	SGEMM_CN		sgemm_tn
 #define	SGEMM_TN		sgemm_tn

common_sb.h

@@ -24,6 +24,7 @@
 #define	SBGEMM_BETA		sbgemm_beta
 #define SBGEMM_KERNEL            sbgemm_kernel
 
+#define SBGEMM_SMALL_MATRIX_PERMIT	sbgemm_small_matrix_permit
 #else
 
 #define SBDOT_K             gotoblas -> sbdot_k
@@ -41,8 +42,19 @@
 #define	SBGEMM_BETA		gotoblas -> sbgemm_beta
 #define	SBGEMM_KERNEL		gotoblas -> sbgemm_kernel
 
+#define SBGEMM_SMALL_MATRIX_PERMIT	gotoblas -> sbgemm_small_matrix_permit
 #endif
 
+#define SBGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(sbgemm_small_kernel_nn)
+#define SBGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(sbgemm_small_kernel_nt)
+#define SBGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(sbgemm_small_kernel_tn)
+#define SBGEMM_SMALL_KERNEL_TT		FUNC_OFFSET(sbgemm_small_kernel_tt)
+
+#define SBGEMM_SMALL_KERNEL_B0_NN	FUNC_OFFSET(sbgemm_small_kernel_b0_nn)
+#define SBGEMM_SMALL_KERNEL_B0_NT	FUNC_OFFSET(sbgemm_small_kernel_b0_nt)
+#define SBGEMM_SMALL_KERNEL_B0_TN	FUNC_OFFSET(sbgemm_small_kernel_b0_tn)
+#define SBGEMM_SMALL_KERNEL_B0_TT	FUNC_OFFSET(sbgemm_small_kernel_b0_tt)
+
 #define	SBGEMM_NN		sbgemm_nn
 #define	SBGEMM_CN		sbgemm_tn
 #define	SBGEMM_TN		sbgemm_tn

common_z.h

@@ -232,6 +232,8 @@
 
 #define ZGEADD_K                zgeadd_k 
 
+#define ZGEMM_SMALL_MATRIX_PERMIT	zgemm_small_matrix_permit
+
 #else
 
 #define	ZAMAX_K			gotoblas -> zamax_k
@@ -426,8 +428,51 @@
 
 #define ZGEADD_K                gotoblas -> zgeadd_k
 
+#define ZGEMM_SMALL_MATRIX_PERMIT	gotoblas -> zgemm_small_matrix_permit
+
 #endif
 
+#define ZGEMM_SMALL_KERNEL_NN		FUNC_OFFSET(zgemm_small_kernel_nn)
+#define ZGEMM_SMALL_KERNEL_NT		FUNC_OFFSET(zgemm_small_kernel_nt)
+#define ZGEMM_SMALL_KERNEL_NR		FUNC_OFFSET(zgemm_small_kernel_nr)
+#define ZGEMM_SMALL_KERNEL_NC		FUNC_OFFSET(zgemm_small_kernel_nc)
+
+#define ZGEMM_SMALL_KERNEL_TN		FUNC_OFFSET(zgemm_small_kernel_tn)
+#define ZGEMM_SMALL_KERNEL_TT		FUNC_OFFSET(zgemm_small_kernel_tt)
+#define ZGEMM_SMALL_KERNEL_TR		FUNC_OFFSET(zgemm_small_kernel_tr)
+#define ZGEMM_SMALL_KERNEL_TC		FUNC_OFFSET(zgemm_small_kernel_tc)
+
+#define ZGEMM_SMALL_KERNEL_RN		FUNC_OFFSET(zgemm_small_kernel_rn)
+#define ZGEMM_SMALL_KERNEL_RT		FUNC_OFFSET(zgemm_small_kernel_rt)
+#define ZGEMM_SMALL_KERNEL_RR		FUNC_OFFSET(zgemm_small_kernel_rr)
+#define ZGEMM_SMALL_KERNEL_RC		FUNC_OFFSET(zgemm_small_kernel_rc)
+
+#define ZGEMM_SMALL_KERNEL_CN		FUNC_OFFSET(zgemm_small_kernel_cn)
+#define ZGEMM_SMALL_KERNEL_CT		FUNC_OFFSET(zgemm_small_kernel_ct)
+#define ZGEMM_SMALL_KERNEL_CR		FUNC_OFFSET(zgemm_small_kernel_cr)
+#define ZGEMM_SMALL_KERNEL_CC		FUNC_OFFSET(zgemm_small_kernel_cc)
+
+#define ZGEMM_SMALL_KERNEL_B0_NN	FUNC_OFFSET(zgemm_small_kernel_b0_nn)
+#define ZGEMM_SMALL_KERNEL_B0_NT	FUNC_OFFSET(zgemm_small_kernel_b0_nt)
+#define ZGEMM_SMALL_KERNEL_B0_NR	FUNC_OFFSET(zgemm_small_kernel_b0_nr)
+#define ZGEMM_SMALL_KERNEL_B0_NC	FUNC_OFFSET(zgemm_small_kernel_b0_nc)
+
+#define ZGEMM_SMALL_KERNEL_B0_TN	FUNC_OFFSET(zgemm_small_kernel_b0_tn)
+#define ZGEMM_SMALL_KERNEL_B0_TT	FUNC_OFFSET(zgemm_small_kernel_b0_tt)
+#define ZGEMM_SMALL_KERNEL_B0_TR	FUNC_OFFSET(zgemm_small_kernel_b0_tr)
+#define ZGEMM_SMALL_KERNEL_B0_TC	FUNC_OFFSET(zgemm_small_kernel_b0_tc)
+
+#define ZGEMM_SMALL_KERNEL_B0_RN	FUNC_OFFSET(zgemm_small_kernel_b0_rn)
+#define ZGEMM_SMALL_KERNEL_B0_RT	FUNC_OFFSET(zgemm_small_kernel_b0_rt)
+#define ZGEMM_SMALL_KERNEL_B0_RR	FUNC_OFFSET(zgemm_small_kernel_b0_rr)
+#define ZGEMM_SMALL_KERNEL_B0_RC	FUNC_OFFSET(zgemm_small_kernel_b0_rc)
+
+#define ZGEMM_SMALL_KERNEL_B0_CN	FUNC_OFFSET(zgemm_small_kernel_b0_cn)
+#define ZGEMM_SMALL_KERNEL_B0_CT	FUNC_OFFSET(zgemm_small_kernel_b0_ct)
+#define ZGEMM_SMALL_KERNEL_B0_CR	FUNC_OFFSET(zgemm_small_kernel_b0_cr)
+#define ZGEMM_SMALL_KERNEL_B0_CC	FUNC_OFFSET(zgemm_small_kernel_b0_cc)
+
+
 #define	ZGEMM_NN		zgemm_nn
 #define	ZGEMM_CN		zgemm_cn
 #define	ZGEMM_TN		zgemm_tn

ctest.c

@@ -84,7 +84,7 @@ OS_AIX
 OS_OSF
 #endif
 
-#if defined(__WIN32) || defined(__WIN64) || defined(__WINNT)
+#if defined(__WIN32) || defined(__WIN64) || defined(_WIN32) || defined(_WIN64) || defined(__WINNT)
 OS_WINNT
 #endif
 
@@ -141,7 +141,7 @@ ARCH_SPARC
 ARCH_IA64
 #endif
 
-#if defined(__LP64) || defined(__LP64__) || defined(__ptr64) || defined(__x86_64__) || defined(__amd64__) || defined(__64BIT__)
+#if defined(__LP64) || defined(__LP64__) || defined(__ptr64) || defined(__x86_64__) || defined(__amd64__) || defined(__64BIT__) || defined(__aarch64__)
 BINARY_64
 #endif
 

driver/level2/CMakeLists.txt

@@ -81,6 +81,7 @@ foreach (float_type ${FLOAT_TYPES})
     GenerateNamedObjects("gbmv_thread.c" "TRANSA" "gbmv_thread_t" false "" "" false ${float_type})
   endif ()
 
+# special defines for complex
   if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
 
     foreach (u_source ${U_SOURCES})
@@ -197,6 +198,13 @@ foreach (float_type ${FLOAT_TYPES})
   endif ()
 endforeach ()
 
+if (BUILD_BFLOAT16)
+  if (USE_THREAD)
+    GenerateNamedObjects("sbgemv_thread.c" "" "gemv_thread_n" false "" "" false "BFLOAT16")
+    GenerateNamedObjects("sbgemv_thread.c" "TRANSA" "gemv_thread_t" false "" "" false "BFLOAT16")
+  endif ()
+endif ()
+
 if ( BUILD_COMPLEX AND NOT  BUILD_SINGLE)
   if (USE_THREAD)
 	  GenerateNamedObjects("gemv_thread.c" "" "gemv_thread_n" false "" "" false "SINGLE")

driver/level3/CMakeLists.txt

@@ -12,6 +12,12 @@ foreach (GEMM_DEFINE ${GEMM_DEFINES})
   if (USE_THREAD AND NOT USE_SIMPLE_THREADED_LEVEL3)
     GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3" "gemm_thread_${GEMM_DEFINE_LC}" 0)
   endif ()
+  if (BUILD_BFLOAT16)
+    GenerateNamedObjects("gemm.c" "${GEMM_DEFINE}" "gemm_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
+    if (USE_THREAD AND NOT USE_SIMPLE_THREADED_LEVEL3)
+      GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3" "gemm_thread_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
+    endif ()
+  endif ()
 endforeach ()
 
 if ( BUILD_COMPLEX16 AND NOT  BUILD_DOUBLE)

driver/others/dynamic_power.c

@@ -6,10 +6,6 @@ extern gotoblas_t gotoblas_POWER8;
 #if (!defined __GNUC__) || ( __GNUC__ >= 6)
 extern gotoblas_t gotoblas_POWER9;
 #endif
-//#if (!defined __GNUC__) || ( __GNUC__ >= 11) \
-//     || (__GNUC__ == 10 && __GNUC_MINOR__ >= 2)
-//#define HAVE_P10_SUPPORT 1
-//#endif
 #ifdef HAVE_P10_SUPPORT
 extern gotoblas_t gotoblas_POWER10;
 #endif

driver/others/memory.c

@@ -73,6 +73,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
+#ifndef likely
+#ifdef __GNUC__
+#define likely(x) __builtin_expect(!!(x), 1)
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#else
+#define likely(x) (x)
+#define unlikely(x) (x)
+#endif
+#endif
+
 #if defined(USE_TLS) && defined(SMP)
 #define COMPILE_TLS
 
@@ -2060,6 +2070,7 @@ struct release_t {
 int hugetlb_allocated = 0;
 
 static struct release_t release_info[NUM_BUFFERS];
+static struct release_t *new_release_info;
 static int release_pos = 0;
 
 #if defined(OS_LINUX) && !defined(NO_WARMUP)
@@ -2110,8 +2121,13 @@ static void *alloc_mmap(void *address){
 #if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
     LOCK_COMMAND(&alloc_lock);
 #endif
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_mmap_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_mmap_free;
+    }
     release_pos ++;
 #if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
     UNLOCK_COMMAND(&alloc_lock);
@@ -2274,8 +2290,13 @@ static void *alloc_mmap(void *address){
 #if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
     LOCK_COMMAND(&alloc_lock);
 #endif
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_mmap_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_mmap_free;
+    }
     release_pos ++;
 #if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
     UNLOCK_COMMAND(&alloc_lock);
@@ -2307,8 +2328,13 @@ static void *alloc_malloc(void *address){
   if (map_address == (void *)NULL) map_address = (void *)-1;
 
   if (map_address != (void *)-1) {
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_malloc_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_malloc_free;
+    }
     release_pos ++;
   }
 
@@ -2341,8 +2367,13 @@ static void *alloc_qalloc(void *address){
   if (map_address == (void *)NULL) map_address = (void *)-1;
 
   if (map_address != (void *)-1) {
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_qalloc_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_qalloc_free;
+    }
     release_pos ++;
   }
 
@@ -2370,8 +2401,13 @@ static void *alloc_windows(void *address){
   if (map_address == (void *)NULL) map_address = (void *)-1;
 
   if (map_address != (void *)-1) {
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_windows_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_windows_free;
+    }
     release_pos ++;
   }
 
@@ -2414,9 +2450,15 @@ static void *alloc_devicedirver(void *address){
                      fd, 0);
 
   if (map_address != (void *)-1) {
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].attr    = fd;
     release_info[release_pos].func    = alloc_devicedirver_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].attr    = fd;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_devicedirver_free;
+    }
     release_pos ++;
   }
 
@@ -2450,9 +2492,15 @@ static void *alloc_shm(void *address){
 
     shmctl(shmid, IPC_RMID, 0);
 
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].attr    = shmid;
     release_info[release_pos].func    = alloc_shm_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].attr    = shmid;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_shm_free;
+    }
     release_pos ++;
   }
 
@@ -2556,8 +2604,13 @@ static void *alloc_hugetlb(void *address){
 #endif
 
   if (map_address != (void *)-1){
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].func    = alloc_hugetlb_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_hugetlb_free;
+    }
     release_pos ++;
   }
 
@@ -2604,9 +2657,15 @@ static void *alloc_hugetlbfile(void *address){
                      fd, 0);
 
   if (map_address != (void *)-1) {
+    if (likely(release_pos < NUM_BUFFERS)) {
     release_info[release_pos].address = map_address;
     release_info[release_pos].attr    = fd;
     release_info[release_pos].func    = alloc_hugetlbfile_free;
+    } else {
+    new_release_info[release_pos-NUM_BUFFERS].address = map_address;
+    new_release_info[release_pos-NUM_BUFFERS].attr    = fd;
+    new_release_info[release_pos-NUM_BUFFERS].func    = alloc_hugetlbfile_free;
+    }
     release_pos ++;
   }
 
@@ -2636,8 +2695,25 @@ static volatile struct {
 
 } memory[NUM_BUFFERS];
 
-static int memory_initialized = 0;
+struct newmemstruct 
+{
+  BLASULONG lock;
+  void *addr;
+#if defined(WHEREAMI) && !defined(USE_OPENMP)
+  int   pos;
+#endif
+  int used;
+#ifndef __64BIT__
+  char dummy[48];
+#else
+  char dummy[40];
+#endif
 
+};
+static volatile struct newmemstruct *newmemory;
+
+static int memory_initialized = 0;
+static int memory_overflowed = 0;
 /*       Memory allocation routine           */
 /* procpos ... indicates where it comes from */
 /*                0 : Level 3 functions      */
@@ -2646,6 +2722,8 @@ static int memory_initialized = 0;
 
 void *blas_memory_alloc(int procpos){
 
+  int i;
+  
   int position;
 #if defined(WHEREAMI) && !defined(USE_OPENMP)
   int mypos = 0;
@@ -2779,6 +2857,29 @@ void *blas_memory_alloc(int procpos){
 #if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
   UNLOCK_COMMAND(&alloc_lock);
 #endif
+  if (memory_overflowed) {
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+  LOCK_COMMAND(&alloc_lock);
+#endif
+  do {
+    RMB;
+#if defined(USE_OPENMP)
+    if (!newmemory[position-NUM_BUFFERS].used) {
+      blas_lock(&newmemory[position-NUM_BUFFERS].lock);
+#endif
+      if (!newmemory[position-NUM_BUFFERS].used) goto allocation2;
+
+#if defined(USE_OPENMP)
+      blas_unlock(&newmemory[position-NUM_BUFFERS].lock);
+    }
+#endif
+    position ++;
+
+  } while (position < 512+NUM_BUFFERS);
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+  UNLOCK_COMMAND(&alloc_lock);
+#endif
+} 
   goto error;
 
   allocation :
@@ -2883,6 +2984,91 @@ void *blas_memory_alloc(int procpos){
   return (void *)memory[position].addr;
 
  error:
+ if (memory_overflowed) goto terminate;
+  fprintf(stderr,"OpenBLAS warning: precompiled NUM_THREADS exceeded, adding auxiliary array for thread metadata.\n");
+  memory_overflowed=1;
+  new_release_info = (struct release_t*) malloc(512*sizeof(struct release_t));
+  newmemory = (struct newmemstruct*) malloc(512*sizeof(struct newmemstruct));
+  for (i = 0; i < 512; i++) {
+  newmemory[i].addr   = (void *)0;
+#if defined(WHEREAMI) && !defined(USE_OPENMP)
+  newmemory[i].pos    = -1;
+#endif
+  newmemory[i].used   = 0;
+  newmemory[i].lock   = 0;
+}
+  newmemory[position-NUM_BUFFERS].used = 1;
+  
+allocation2:
+  newmemory[position-NUM_BUFFERS].used = 1;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+  UNLOCK_COMMAND(&alloc_lock);
+#else
+  blas_unlock(&newmemory[position-NUM_BUFFERS].lock);
+#endif
+    do {
+#ifdef DEBUG
+      printf("Allocation Start : %lx\n", base_address);
+#endif
+
+      map_address = (void *)-1;
+
+      func = &memoryalloc[0];
+
+      while ((func != NULL) && (map_address == (void *) -1)) {
+
+        map_address = (*func)((void *)base_address);
+
+#ifdef ALLOC_DEVICEDRIVER
+        if ((*func ==  alloc_devicedirver) && (map_address == (void *)-1)) {
+            fprintf(stderr, "OpenBLAS Warning ... Physically contiguous allocation was failed.\n");
+        }
+#endif
+
+#ifdef ALLOC_HUGETLBFILE
+        if ((*func == alloc_hugetlbfile) && (map_address == (void *)-1)) {
+#ifndef OS_WINDOWS
+            fprintf(stderr, "OpenBLAS Warning ... HugeTLB(File) allocation was failed.\n");
+#endif
+        }
+#endif
+
+#if (defined ALLOC_SHM) && (defined OS_LINUX  || defined OS_AIX  || defined __sun__  || defined OS_WINDOWS)
+        if ((*func == alloc_hugetlb) && (map_address != (void *)-1)) hugetlb_allocated = 1;
+#endif
+
+        func ++;
+      }
+
+#ifdef DEBUG
+      printf("  Success -> %08lx\n", map_address);
+#endif
+      if (((BLASLONG) map_address) == -1) base_address = 0UL;
+
+      if (base_address) base_address += BUFFER_SIZE + FIXED_PAGESIZE;
+
+    } while ((BLASLONG)map_address == -1);
+
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+    LOCK_COMMAND(&alloc_lock);
+#endif
+    newmemory[position-NUM_BUFFERS].addr = map_address;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+    UNLOCK_COMMAND(&alloc_lock);
+#endif
+
+#ifdef DEBUG
+    printf("  Mapping Succeeded. %p(%d)\n", (void *)newmemory[position-NUM_BUFFERS].addr, position);
+#endif
+
+#if defined(WHEREAMI) && !defined(USE_OPENMP)
+
+  if (newmemory[position-NUM_BUFFERS].pos == -1) newmemory[position-NUM_BUFFERS].pos = mypos;
+
+#endif
+  return (void *)newmemory[position-NUM_BUFFERS].addr;
+
+terminate:
   printf("OpenBLAS : Program is Terminated. Because you tried to allocate too many memory regions.\n");
   printf("This library was built to support a maximum of %d threads - either rebuild OpenBLAS\n", NUM_BUFFERS);
   printf("with a larger NUM_THREADS value or set the environment variable OPENBLAS_NUM_THREADS to\n");
@@ -2907,13 +3093,28 @@ void blas_memory_free(void *free_area){
   while ((position < NUM_BUFFERS) && (memory[position].addr != free_area))
     position++;
 
-  if (position >= NUM_BUFFERS) goto error;
+  if (position >= NUM_BUFFERS && !memory_overflowed) goto error;
 
 #ifdef DEBUG
   if (memory[position].addr != free_area) goto error;
   printf("  Position : %d\n", position);
 #endif
+  if (unlikely(memory_overflowed && position >= NUM_BUFFERS)) {
+    while ((position < NUM_BUFFERS+512) && (newmemory[position-NUM_BUFFERS].addr != free_area))
+      position++;
+  // arm: ensure all writes are finished before other thread takes this memory
+  WMB;
 
+  newmemory[position].used = 0;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+  UNLOCK_COMMAND(&alloc_lock);
+#endif
+
+#ifdef DEBUG
+  printf("Unmap from overflow area succeeded.\n\n");
+#endif
+  return;
+} else {
   // arm: ensure all writes are finished before other thread takes this memory
   WMB;
 
@@ -2927,7 +3128,7 @@ void blas_memory_free(void *free_area){
 #endif
 
   return;
-
+}
  error:
   printf("BLAS : Bad memory unallocation! : %4d  %p\n", position,  free_area);
 
@@ -2962,7 +3163,10 @@ void blas_shutdown(void){
   LOCK_COMMAND(&alloc_lock);
 
   for (pos = 0; pos < release_pos; pos ++) {
+    if (likely(pos < NUM_BUFFERS))
     release_info[pos].func(&release_info[pos]);
+    else
+    new_release_info[pos-NUM_BUFFERS].func(&new_release_info[pos-NUM_BUFFERS]);
   }
 
 #ifdef SEEK_ADDRESS
@@ -2979,6 +3183,15 @@ void blas_shutdown(void){
 #endif
     memory[pos].lock   = 0;
   }
+  if (memory_overflowed)
+    for (pos = 0; pos < 512; pos ++){
+      newmemory[pos].addr   = (void *)0;
+      newmemory[pos].used   = 0;
+#if defined(WHEREAMI) && !defined(USE_OPENMP)
+      newmemory[pos].pos    = -1;
+#endif
+      newmemory[pos].lock   = 0;
+  }
 
   UNLOCK_COMMAND(&alloc_lock);
 

driver/others/parameter.c

@@ -524,6 +524,9 @@ void blas_set_parameter(void){
   xgemm_p = ((xgemm_p + XGEMM_UNROLL_M - 1)/XGEMM_UNROLL_M) * XGEMM_UNROLL_M;
 #endif
 
+#ifdef BUILD_BFLOAT16
+  sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q *  4)) - 15) & ~15;
+#endif
   sgemm_r = (((BUFFER_SIZE - ((SGEMM_P * SGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SGEMM_Q *  4)) - 15) & ~15;
   dgemm_r = (((BUFFER_SIZE - ((DGEMM_P * DGEMM_Q *  8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (DGEMM_Q *  8)) - 15) & ~15;
   cgemm_r = (((BUFFER_SIZE - ((CGEMM_P * CGEMM_Q *  8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (CGEMM_Q *  8)) - 15) & ~15;
@@ -629,7 +632,9 @@ void blas_set_parameter(void){
   xgemm_p =  16 * (size + 1);
 #endif
 
+#ifdef BUILD_BFLOAT16
   sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q *  4)) - 15) & ~15;
+#endif
   sgemm_r = (((BUFFER_SIZE - ((SGEMM_P * SGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SGEMM_Q *  4)) - 15) & ~15;
   dgemm_r = (((BUFFER_SIZE - ((DGEMM_P * DGEMM_Q *  8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (DGEMM_Q *  8)) - 15) & ~15;
   cgemm_r = (((BUFFER_SIZE - ((CGEMM_P * CGEMM_Q *  8 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (CGEMM_Q *  8)) - 15) & ~15;

getarch.c

@@ -313,6 +313,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define FORCE
 #define FORCE_INTEL
 #define ARCHITECTURE    "X86"
+#ifdef NO_AVX 
+#define SUBARCHITECTURE "NEHALEM"
+#define ARCHCONFIG   "-DNEHALEM " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2"
+#define LIBNAME   "nehalem"
+#define CORENAME  "NEHALEM"
+#else
 #define SUBARCHITECTURE "SANDYBRIDGE"
 #define ARCHCONFIG   "-DSANDYBRIDGE " \
 		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -322,12 +332,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIBNAME   "sandybridge"
 #define CORENAME  "SANDYBRIDGE"
 #endif
+#endif
 
 #ifdef FORCE_HASWELL
 #define FORCE
 #define FORCE_INTEL
 #define ARCHITECTURE    "X86"
 #ifdef NO_AVX2
+#ifdef NO_AVX
+#define SUBARCHITECTURE "NEHALEM"
+#define ARCHCONFIG   "-DNEHALEM " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2"
+#define LIBNAME   "nehalem"
+#define CORENAME  "NEHALEM"
+#else
 #define SUBARCHITECTURE "SANDYBRIDGE"
 #define ARCHCONFIG   "-DSANDYBRIDGE " \
 		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -336,6 +357,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 -DHAVE_AVX"
 #define LIBNAME   "sandybridge"
 #define CORENAME  "SANDYBRIDGE"
+#endif
 #else
 #define SUBARCHITECTURE "HASWELL"
 #define ARCHCONFIG   "-DHASWELL " \
@@ -350,10 +372,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #ifdef FORCE_SKYLAKEX
-#ifdef NO_AVX512
 #define FORCE
 #define FORCE_INTEL
 #define ARCHITECTURE    "X86"
+#ifdef NO_AVX512
+#ifdef NO_AVX2
+#ifdef NO_AVX
+#define SUBARCHITECTURE "NEHALEM"
+#define ARCHCONFIG   "-DNEHALEM " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2"
+#define LIBNAME   "nehalem"
+#define CORENAME  "NEHALEM"
+#else
+#define SUBARCHITECTURE "SANDYBRIDGE"
+#define ARCHCONFIG   "-DSANDYBRIDGE " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 -DHAVE_AVX"
+#define LIBNAME   "sandybridge"
+#define CORENAME  "SANDYBRIDGE"
+#endif
+#else
 #define SUBARCHITECTURE "HASWELL"
 #define ARCHCONFIG   "-DHASWELL " \
 		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -363,10 +406,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                      "-DHAVE_AVX2 -DHAVE_FMA3 -DFMA3"
 #define LIBNAME   "haswell"
 #define CORENAME  "HASWELL"
+#endif
 #else
-#define FORCE
-#define FORCE_INTEL
-#define ARCHITECTURE    "X86"
 #define SUBARCHITECTURE "SKYLAKEX"
 #define ARCHCONFIG   "-DSKYLAKEX " \
 		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -380,10 +421,31 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #ifdef FORCE_COOPERLAKE
-#ifdef NO_AVX512
 #define FORCE
 #define FORCE_INTEL
 #define ARCHITECTURE    "X86"
+#ifdef NO_AVX512
+#ifdef NO_AVX2
+#ifdef NO_AVX
+#define SUBARCHITECTURE "NEHALEM"
+#define ARCHCONFIG   "-DNEHALEM " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2"
+#define LIBNAME   "nehalem"
+#define CORENAME  "NEHALEM"
+#else
+#define SUBARCHITECTURE "SANDYBRIDGE"
+#define ARCHCONFIG   "-DSANDYBRIDGE " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 -DHAVE_AVX"
+#define LIBNAME   "sandybridge"
+#define CORENAME  "SANDYBRIDGE"
+#endif
+#else
 #define SUBARCHITECTURE "HASWELL"
 #define ARCHCONFIG   "-DHASWELL " \
                      "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -393,10 +455,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                      "-DHAVE_AVX2 -DHAVE_FMA3 -DFMA3"
 #define LIBNAME   "haswell"
 #define CORENAME  "HASWELL"
+#endif
 #else
-#define FORCE
-#define FORCE_INTEL
-#define ARCHITECTURE    "X86"
 #define SUBARCHITECTURE "COOPERLAKE"
 #define ARCHCONFIG   "-DCOOPERLAKE " \
                      "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -564,6 +624,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define FORCE_INTEL
 #define ARCHITECTURE    "X86"
 #ifdef NO_AVX2
+#ifdef NO_AVX
+#define SUBARCHITECTURE "NEHALEM"
+#define ARCHCONFIG   "-DNEHALEM " \
+		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
+		     "-DL2_SIZE=262144 -DL2_LINESIZE=64 " \
+		     "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
+		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2"
+#define LIBNAME   "nehalem"
+#define CORENAME  "NEHALEM"
+#else
 #define SUBARCHITECTURE "SANDYBRIDGE"
 #define ARCHCONFIG   "-DSANDYBRIDGE " \
 		     "-DL1_DATA_SIZE=32768 -DL1_DATA_LINESIZE=64 " \
@@ -572,6 +642,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 		     "-DHAVE_CMOV -DHAVE_MMX -DHAVE_SSE -DHAVE_SSE2 -DHAVE_SSE3 -DHAVE_SSSE3 -DHAVE_SSE4_1 -DHAVE_SSE4_2 -DHAVE_AVX"
 #define LIBNAME   "sandybridge"
 #define CORENAME  "SANDYBRIDGE"
+#endif
 #else
 #define SUBARCHITECTURE "ZEN"
 #define ARCHCONFIG   "-DZEN " \

interface/CMakeLists.txt

@@ -82,6 +82,7 @@ foreach (CBLAS_FLAG ${CBLAS_FLAGS})
   GenerateNamedObjects("${BLAS3_SOURCES}" "" "" ${CBLAS_FLAG} "" "" false ${DISABLE_COMPLEX})
   GenerateNamedObjects("${BLAS3_MANGLED_SOURCES}" "" "" ${CBLAS_FLAG} "" "" false ${MANGLE_COMPLEX})
 
+  GenerateNamedObjects("xerbla.c" "" "xerbla" ${CBLAS_FLAG} "" "" true)
   #sdsdot, dsdot
   if (BUILD_SINGLE OR BUILD_DOUBLE)
   GenerateNamedObjects("sdsdot.c" "" "sdsdot" ${CBLAS_FLAG} "" "" true "SINGLE")
@@ -104,6 +105,15 @@ endif ()
   GenerateNamedObjects("imax.c" "USE_ABS;USE_MIN" "i*amin" ${CBLAS_FLAG})
   GenerateNamedObjects("imax.c" "USE_MIN" "i*min" ${CBLAS_FLAG})
 
+if (BUILD_BFLOAT16)
+	GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("bf16to.c" "SINGLE_PREC" "sbf16tos" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("bf16to.c" "DOUBLE_PREC" "dbf16tod" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+endif ()
 
 # complex-specific sources
 foreach (float_type ${FLOAT_TYPES})

interface/gemm.c

@@ -105,6 +105,55 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B
 #endif
 };
 
+#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE)
+#define USE_SMALL_MATRIX_OPT 1
+#else
+#define USE_SMALL_MATRIX_OPT 0
+#endif
+
+#if USE_SMALL_MATRIX_OPT
+#ifndef DYNAMIC_ARCH
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx]))
+#else
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx]))))
+#endif
+
+
+#ifndef COMPLEX
+static size_t gemm_small_kernel[] = {
+	GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, 0, 0,
+	GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, 0, 0,
+};
+
+
+static size_t gemm_small_kernel_b0[] = {
+	GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, 0, 0,
+	GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, 0, 0,
+};
+
+#define GEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel_b0, (idx))
+#define GEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel, (idx))
+#else
+
+static size_t zgemm_small_kernel[] = {
+	GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, GEMM_SMALL_KERNEL_RN, GEMM_SMALL_KERNEL_CN,
+	GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, GEMM_SMALL_KERNEL_RT, GEMM_SMALL_KERNEL_CT,
+	GEMM_SMALL_KERNEL_NR, GEMM_SMALL_KERNEL_TR, GEMM_SMALL_KERNEL_RR, GEMM_SMALL_KERNEL_CR,
+	GEMM_SMALL_KERNEL_NC, GEMM_SMALL_KERNEL_TC, GEMM_SMALL_KERNEL_RC, GEMM_SMALL_KERNEL_CC,
+};
+
+static size_t zgemm_small_kernel_b0[] = {
+	GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, GEMM_SMALL_KERNEL_B0_RN, GEMM_SMALL_KERNEL_B0_CN,
+	GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, GEMM_SMALL_KERNEL_B0_RT, GEMM_SMALL_KERNEL_B0_CT,
+	GEMM_SMALL_KERNEL_B0_NR, GEMM_SMALL_KERNEL_B0_TR, GEMM_SMALL_KERNEL_B0_RR, GEMM_SMALL_KERNEL_B0_CR,
+	GEMM_SMALL_KERNEL_B0_NC, GEMM_SMALL_KERNEL_B0_TC, GEMM_SMALL_KERNEL_B0_RC, GEMM_SMALL_KERNEL_B0_CC,
+};
+
+#define ZGEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel, (idx))
+#define ZGEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel_b0, (idx))
+#endif
+#endif
+
 #ifndef CBLAS
 
 void NAME(char *TRANSA, char *TRANSB,
@@ -224,8 +273,8 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
 	   blasint m, blasint n, blasint k,
 #ifndef COMPLEX
 	   FLOAT alpha,
-	   FLOAT *a, blasint lda,
-	   FLOAT *b, blasint ldb,
+	   IFLOAT *a, blasint lda,
+	   IFLOAT *b, blasint ldb,
 	   FLOAT beta,
 	   FLOAT *c, blasint ldc) {
 #else
@@ -277,7 +326,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
 
   PRINT_DEBUG_CNAME;
 
-#if !defined(COMPLEX) && !defined(DOUBLE) && defined(USE_SGEMM_KERNEL_DIRECT)
+#if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && defined(USE_SGEMM_KERNEL_DIRECT)
 #ifdef DYNAMIC_ARCH
  if (support_avx512() )
 #endif  
@@ -417,6 +466,28 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
 
   FUNCTION_PROFILE_START();
 
+#if USE_SMALL_MATRIX_OPT
+#if !defined(COMPLEX)
+  if(GEMM_SMALL_MATRIX_PERMIT(transa, transb, args.m, args.n, args.k, *(FLOAT *)(args.alpha), *(FLOAT *)(args.beta))){
+	  if(*(FLOAT *)(args.beta) == 0.0){
+		(GEMM_SMALL_KERNEL_B0((transb << 2) | transa))(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, args.c, args.ldc);
+	  }else{
+		(GEMM_SMALL_KERNEL((transb << 2) | transa))(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, *(FLOAT *)(args.beta), args.c, args.ldc);
+	  }
+	  return;
+  }
+#else
+  if(GEMM_SMALL_MATRIX_PERMIT(transa, transb, args.m, args.n, args.k, alpha[0], alpha[1], beta[0], beta[1])){
+	  if(beta[0] == 0.0 && beta[1] == 0.0){
+		(ZGEMM_SMALL_KERNEL_B0((transb << 2) | transa))(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, args.c, args.ldc);
+	  }else{
+		(ZGEMM_SMALL_KERNEL((transb << 2) | transa))(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, beta[0], beta[1], args.c, args.ldc);
+	  }
+	  return;
+  }
+#endif
+#endif
+
   buffer = (XFLOAT *)blas_memory_alloc(0);
 
   sa = (XFLOAT *)((BLASLONG)buffer +GEMM_OFFSET_A);

interface/zsyr.c

@@ -119,7 +119,7 @@ void NAME(char *UPLO, blasint *N, FLOAT  *ALPHA,
 void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, int n, FLOAT alpha, FLOAT *x, int incx, FLOAT *a, int lda) {
 
   FLOAT *buffer;
-  int trans, uplo;
+  int uplo;
   blasint info;
   FLOAT * ALPHA = α
   FLOAT alpha_r	= ALPHA[0];
@@ -130,7 +130,6 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, int n, FLOAT alpha, FLO
 
   PRINT_DEBUG_CNAME;
 
-  trans = -1;
   uplo  = -1;
   info  =  0;
 

kernel/CMakeLists.txt

@@ -91,6 +91,15 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     GenerateNamedObjects("${KERNELDIR}/${DSDOTKERNEL}" "DSDOT" "d*dot_k" false "" "" false "SINGLE")
     GenerateNamedObjects("${KERNELDIR}/${DSDOTKERNEL}" "DSDOT" "dsdot_k" false "" "" false "SINGLE")
 
+    # sbdot
+    if (BUILD_BFLOAT16)
+	    GenerateNamedObjects("${KERNELDIR}/${SBDOTKERNEL}" "SBDOT" "dot_k" false "" "" false "BFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "SINGLE" "f16tos_k" false "" "" false "BFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "DOUBLE" "bf16tod_k" false "" "" false "DOUBLE")
+	    GenerateNamedObjects("${KERNELDIR}/${TOBF16KERNEL}" "SINGLE" "stobf16_k" false "" "" false "BFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${TOBF16KERNEL}" "DOUBLE" "dtobf16_k" false "" "" false "BFLOAT16")
+    endif()
+
     if ((BUILD_COMPLEX OR BUILD_DOUBLE)  AND NOT BUILD_SINGLE)
     GenerateNamedObjects("${KERNELDIR}/${SAMAXKERNEL}" "USE_ABS" "amax_k" false "" "" false "SINGLE")
     GenerateNamedObjects("${KERNELDIR}/${SAMINKERNEL}" "USE_ABS;USE_MIN" "amin_k" false "" "" false "SINGLE")
@@ -149,9 +158,6 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     GenerateNamedObjects("generic/ger.c" "" "ger_k" false "" "" "" 3)
     foreach (float_type ${FLOAT_TYPES})
       string(SUBSTRING ${float_type} 0 1 float_char)
-      if (${float_type} STREQUAL "BFLOAT16")
-	set (float_char "SB")
-      endif ()
       if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
         GenerateNamedObjects("${KERNELDIR}/${${float_char}GERUKERNEL}" "" "geru_k" false "" "" false ${float_type})
         GenerateNamedObjects("${KERNELDIR}/${${float_char}GERCKERNEL}" "CONJ" "gerc_k" false "" "" false ${float_type})
@@ -185,6 +191,10 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	    GenerateNamedObjects("${KERNELDIR}/${SGEMVNKERNEL}" "" "gemv_n" false "" "" false "SINGLE")
 	    GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE")
     endif ()
+    if (BUILD_BFLOAT16)
+	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16")
+    endif ()
     # Makefile.L3
     set(USE_TRMM false)
     string(TOUPPER ${TARGET_CORE} UC_TARGET_CORE)
@@ -209,15 +219,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPERFORMANT}" "" "gemm_direct_performant" false "" "" false  SINGLE)
     endif()
 
-    foreach (float_type SINGLE DOUBLE BFLOAT16)
+    foreach (float_type SINGLE DOUBLE)
       string(SUBSTRING ${float_type} 0 1 float_char)
-      if (${float_type} STREQUAL "BFLOAT16")
-        if (NOT ${BUILD_BFLOAT16})
-	  continue ()
-        else ()
-	  set (float_char "SB")
-      endif ()
-      endif ()
       GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type})
     endforeach()
     if (BUILD_COMPLEX16  AND NOT BUILD_DOUBLE)
@@ -253,11 +256,24 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	    GenerateNamedObjects("${KERNELDIR}/${SGEMM_BETA}" "" "gemm_beta" false "" "" false "SINGLE")
     endif ()
 
+    if (BUILD_BFLOAT16)
+        if (SBGEMMINCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${SBGEMMINCOPY}" "" "${SBGEMMINCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (SBGEMMITCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${SBGEMMITCOPY}" "" "${SBGEMMITCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (SBGEMMONCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${SBGEMMONCOPY}" "" "${SBGEMMONCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (SBGEMMOTCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${SBGEMMOTCOPY}" "" "${SBGEMMOTCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+	GenerateNamedObjects("${KERNELDIR}/${SBGEMMKERNEL}" "" "gemm_kernel" false "" "" false "BFLOAT16")
+	GenerateNamedObjects("${KERNELDIR}/${SBGEMM_BETA}" "" "gemm_beta" false "" "" false "BFLOAT16")
+    endif ()
     foreach (float_type ${FLOAT_TYPES})
       string(SUBSTRING ${float_type} 0 1 float_char)
-      if (${float_type} STREQUAL "BFLOAT16")
-	set (float_char "SB")
-      endif ()
       if (${float_char}GEMMINCOPY)
         GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "${${float_char}GEMMINCOPYOBJ}" false "" "" true ${float_type})
       endif ()
@@ -458,7 +474,155 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
       GenerateNamedObjects("${KERNELDIR}/${${float_char}TRSMKERNEL_RN}" "UPPER;RN;TRSMKERNEL" "trsm_kernel_RN" false "" "" false ${float_type})
       GenerateNamedObjects("${KERNELDIR}/${${float_char}TRSMKERNEL_RT}" "RT;TRSMKERNEL" "trsm_kernel_RT" false "" "" false ${float_type})
 
+      if (NOT DEFINED ${float_char}GEMM_SMALL_M_PERMIT)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_M_PERMIT ../generic/zgemm_small_matrix_permit.c)
+        else ()
+          set(${float_char}GEMM_SMALL_M_PERMIT ../generic/gemm_small_matrix_permit.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_NN)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_NN ../generic/zgemm_small_matrix_kernel_nn.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_NN ../generic/gemm_small_matrix_kernel_nn.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_NT)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_NT ../generic/zgemm_small_matrix_kernel_nt.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_NT ../generic/gemm_small_matrix_kernel_nt.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_TN)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_TN ../generic/zgemm_small_matrix_kernel_tn.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_TN ../generic/gemm_small_matrix_kernel_tn.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_TT)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_TT ../generic/zgemm_small_matrix_kernel_tt.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_TT ../generic/gemm_small_matrix_kernel_tt.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_B0_NN)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_B0_NN ../generic/zgemm_small_matrix_kernel_nn.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_B0_NN ../generic/gemm_small_matrix_kernel_nn.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_B0_NT)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_B0_NT ../generic/zgemm_small_matrix_kernel_nt.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_B0_NT ../generic/gemm_small_matrix_kernel_nt.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_B0_TN)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_B0_TN ../generic/zgemm_small_matrix_kernel_tn.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_B0_TN ../generic/gemm_small_matrix_kernel_tn.c)
+        endif ()
+      endif ()
+      if (NOT DEFINED ${float_char}GEMM_SMALL_K_B0_TT)
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+          set(${float_char}GEMM_SMALL_K_B0_TT ../generic/zgemm_small_matrix_kernel_tt.c)
+        else ()
+          set(${float_char}GEMM_SMALL_K_B0_TT ../generic/gemm_small_matrix_kernel_tt.c)
+        endif ()
+      endif ()
 
+      if (SMALL_MATRIX_OPT)
+        GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_M_PERMIT}" "" "gemm_small_matrix_permit" false "" "" false ${float_type})
+        if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NN}" "NN" "gemm_small_kernel_nn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NN}" "NR" "gemm_small_kernel_nr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NN}" "RN" "gemm_small_kernel_rn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NN}" "RR" "gemm_small_kernel_rr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "NT" "gemm_small_kernel_nt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "NC" "gemm_small_kernel_nc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "RT" "gemm_small_kernel_rt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "RC" "gemm_small_kernel_rc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TN}" "TN" "gemm_small_kernel_tn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TN}" "TR" "gemm_small_kernel_tr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TN}" "CN" "gemm_small_kernel_cn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TN}" "CR" "gemm_small_kernel_cr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TT}" "TT" "gemm_small_kernel_tt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TT}" "TC" "gemm_small_kernel_tc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TT}" "CT" "gemm_small_kernel_ct" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TT}" "CC" "gemm_small_kernel_cc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NN}" "NN;B0" "gemm_small_kernel_b0_nn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NN}" "NR;B0" "gemm_small_kernel_b0_nr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NN}" "RN;B0" "gemm_small_kernel_b0_rn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NN}" "RR;B0" "gemm_small_kernel_b0_rr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "NT;B0" "gemm_small_kernel_b0_nt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "NC;B0" "gemm_small_kernel_b0_nc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "RT;B0" "gemm_small_kernel_b0_rt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "RC;B0" "gemm_small_kernel_b0_rc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "TN;B0" "gemm_small_kernel_b0_tn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "TR;B0" "gemm_small_kernel_b0_tr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "CN;B0" "gemm_small_kernel_b0_cn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "CR;B0" "gemm_small_kernel_b0_cr" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TT}" "TT;B0" "gemm_small_kernel_b0_tt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TT}" "TC;B0" "gemm_small_kernel_b0_tc" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TT}" "CT;B0" "gemm_small_kernel_b0_ct" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TT}" "CC;B0" "gemm_small_kernel_b0_cc" false "" "" false ${float_type})
+
+        else ()
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NN}" "" "gemm_small_kernel_nn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "" "gemm_small_kernel_nt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_TN}" "" "gemm_small_kernel_tn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_NT}" "" "gemm_small_kernel_tt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NN}" "B0" "gemm_small_kernel_b0_nn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_nt" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "B0" "gemm_small_kernel_b0_tn" false "" "" false ${float_type})
+            GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_tt" false "" "" false ${float_type})
+        endif ()
+	if (BUILD_BFLOAT16)
+      if (NOT DEFINED SBGEMM_SMALL_M_PERMIT)
+          set(SBGEMM_SMALL_M_PERMIT ../generic/gemm_small_matrix_permit.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_NN)
+          set(SBGEMM_SMALL_K_NN ../generic/gemm_small_matrix_kernel_nn.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_NT)
+          set(SBGEMM_SMALL_K_NT ../generic/gemm_small_matrix_kernel_nt.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_TN)
+          set(SBGEMM_SMALL_K_TN ../generic/gemm_small_matrix_kernel_tn.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_TT)
+          set(SBGEMM_SMALL_K_TT ../generic/gemm_small_matrix_kernel_tt.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_B0_NN)
+          set(SBGEMM_SMALL_K_B0_NN ../generic/gemm_small_matrix_kernel_nn.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_B0_NT)
+          set(SBGEMM_SMALL_K_B0_NT ../generic/gemm_small_matrix_kernel_nt.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_B0_TN)
+          set(SBGEMM_SMALL_K_B0_TN ../generic/gemm_small_matrix_kernel_tn.c)
+      endif ()
+      if (NOT DEFINED SBGEMM_SMALL_K_B0_TT)
+          set($SBGEMM_SMALL_K_B0_TT ../generic/gemm_small_matrix_kernel_tt.c)
+      endif ()
+	    GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_M_PERMIT}" "" "gemm_small_matrix_permit" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NN}" "" "gemm_small_kernel_nn" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NT}" "" "gemm_small_kernel_nt" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_TN}" "" "gemm_small_kernel_tn" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NT}" "" "gemm_small_kernel_tt" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NN}" "B0" "gemm_small_kernel_b0_nn" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_nt" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_TN}" "B0" "gemm_small_kernel_b0_tn" false "" "" false "BFLOAT16")
+            GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_tt" false "" "" false "BFLOAT16")
+        endif ()
+      endif ()
 
       if (NOT DEFINED ${float_char}OMATCOPY_CN)
         if (${float_char} STREQUAL "Z" OR ${float_char} STREQUAL "C")
@@ -592,6 +756,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
       #geadd
       GenerateNamedObjects("${KERNELDIR}/${${float_char}GEADD_KERNEL}" "" "geadd_k" false "" "" false ${float_type})
     endforeach ()
+
     if (BUILD_DOUBLE AND NOT BUILD_SINGLE)
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LN}" "UPPER;LN;TRSMKERNEL" "trsm_kernel_LN" false "" "" false "SINGLE")
 	    GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LT}" "LT;TRSMKERNEL" "trsm_kernel_LT" false "" "" false "SINGLE")
@@ -730,22 +895,22 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
       GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER" "trsm_oltncopy" false "" ${TSUFFIX} false "SINGLE")
 
       if (SGEMMINCOPY)
-		    GenerateNamedObjects("${KERNELDIR}/${SGEMMINCOPY}" "SINGLE" "${SGEMMINCOPYOBJ}" false "" "" true "SINGLE")
+	    GenerateNamedObjects("${KERNELDIR}/${SGEMMINCOPY}" "SINGLE" "${SGEMMINCOPYOBJ}" false "" "" true "SINGLE")
       endif ()
-	    if (SGEMMITCOPY)
-		    GenerateNamedObjects("${KERNELDIR}/${SGEMMITCOPY}" "SINGLE" "${SGEMMITCOPYOBJ}" false "" "" true "SINGLE")
-	    endif ()
-	    if (SGEMMONCOPY)
-		    GenerateNamedObjects("${KERNELDIR}/${SGEMMONCOPY}" "SINGLE" "${SGEMMONCOPYOBJ}" false "" "" true "SINGLE")
-	    endif ()
-	    if (SGEMMOTCOPY)
-		    GenerateNamedObjects("${KERNELDIR}/${SGEMMOTCOPY}" "SINGLE" "${SGEMMOTCOPYOBJ}" false "" "" true "SINGLE")
+      if (SGEMMITCOPY)
+	    GenerateNamedObjects("${KERNELDIR}/${SGEMMITCOPY}" "SINGLE" "${SGEMMITCOPYOBJ}" false "" "" true "SINGLE")
+      endif ()
+      if (SGEMMONCOPY)
+	    GenerateNamedObjects("${KERNELDIR}/${SGEMMONCOPY}" "SINGLE" "${SGEMMONCOPYOBJ}" false "" "" true "SINGLE")
+      endif ()
+      if (SGEMMOTCOPY)
+	    GenerateNamedObjects("${KERNELDIR}/${SGEMMOTCOPY}" "SINGLE" "${SGEMMOTCOPYOBJ}" false "" "" true "SINGLE")
       endif ()
       GenerateNamedObjects("${KERNELDIR}/${SGEMVNKERNEL}" "" "gemv_n" false "" "" false "SINGLE")
       GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE")
     endif ()
-      
-    if (BUILD_COMPLEX16  AND NOT BUILD_DOUBLE)
+
+    if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE)
 	GenerateNamedObjects("generic/neg_tcopy_${DGEMM_UNROLL_M}.c" "" "neg_tcopy" false "" ${TSUFFIX}  false "DOUBLE")
 	GenerateNamedObjects("generic/laswp_ncopy_${DGEMM_UNROLL_N}.c" "" "laswp_ncopy" false "" ${TSUFFIX}  false "DOUBLE")
     endif ()

kernel/Makefile.L3

@@ -447,6 +447,72 @@ XBLASOBJS += \
 
 endif
 
+######  BLAS small matrix optimization #####
+ifeq ($(SMALL_MATRIX_OPT), 1)
+
+ifeq ($(BUILD_BFLOAT16),1)
+SBBLASOBJS += \
+	sbgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \
+	sbgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) sbgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \
+	sbgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) sbgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \
+	sbgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) sbgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \
+	sbgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) sbgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX)
+endif
+
+SBLASOBJS += \
+	sgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \
+	sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \
+	sgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \
+	sgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \
+	sgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX)
+
+DBLASOBJS += \
+	dgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \
+	dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \
+	dgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \
+	dgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \
+	dgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX)
+
+CBLASOBJS += \
+	cgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_tr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_tc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_rn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_rt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_rr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_rc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_cn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_ct$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_cr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_cc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_nr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_nc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_tr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_tc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_rn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_rt$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_rr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_rc$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_cn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_ct$(TSUFFIX).$(SUFFIX) \
+	cgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX)
+
+ZBLASOBJS += \
+	zgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_tr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_tc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_rn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_rt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_rr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_rc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_cn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_ct$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_cr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_cc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_nr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_nc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_tr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_tc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_rn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_rt$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_rr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_rc$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_cn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_ct$(TSUFFIX).$(SUFFIX) \
+	zgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX)
+
+endif
+
 ######  BLAS extensions #####
 
 ifeq ($(BUILD_SINGLE),1)
@@ -4237,3 +4303,469 @@ endif
 $(KDIR)zgeadd_k$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEADD_K)
 	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -UROWM $< -o $@
 
+
+
+######  BLAS small matrix optimization #####
+
+ifndef DGEMM_SMALL_M_PERMIT
+DGEMM_SMALL_M_PERMIT = ../generic/gemm_small_matrix_permit.c
+endif
+
+ifndef DGEMM_SMALL_K_NN
+DGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef DGEMM_SMALL_K_NT
+DGEMM_SMALL_K_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef DGEMM_SMALL_K_TN
+DGEMM_SMALL_K_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef DGEMM_SMALL_K_TT
+DGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)dgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_M_PERMIT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)dgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)dgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)dgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@
+
+ifndef DGEMM_SMALL_K_B0_NN
+DGEMM_SMALL_K_B0_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef DGEMM_SMALL_K_B0_NT
+DGEMM_SMALL_K_B0_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef DGEMM_SMALL_K_B0_TN
+DGEMM_SMALL_K_B0_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef DGEMM_SMALL_K_B0_TT
+DGEMM_SMALL_K_B0_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)dgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)dgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)dgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+ifndef SGEMM_SMALL_M_PERMIT
+SGEMM_SMALL_M_PERMIT = ../generic/gemm_small_matrix_permit.c
+endif
+
+ifndef SGEMM_SMALL_K_NN
+SGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef SGEMM_SMALL_K_NT
+SGEMM_SMALL_K_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef SGEMM_SMALL_K_TN
+SGEMM_SMALL_K_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef SGEMM_SMALL_K_TT
+SGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)sgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_M_PERMIT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+
+ifndef SGEMM_SMALL_K_B0_NN
+SGEMM_SMALL_K_B0_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef SGEMM_SMALL_K_B0_NT
+SGEMM_SMALL_K_B0_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef SGEMM_SMALL_K_B0_TN
+SGEMM_SMALL_K_B0_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef SGEMM_SMALL_K_B0_TT
+SGEMM_SMALL_K_B0_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)sgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+
+ifeq ($(BUILD_BFLOAT16), 1)
+ifndef SBGEMM_SMALL_M_PERMIT
+SBGEMM_SMALL_M_PERMIT = ../generic/gemm_small_matrix_permit.c
+endif
+
+ifndef SBGEMM_SMALL_K_NN
+SBGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef SBGEMM_SMALL_K_NT
+SBGEMM_SMALL_K_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef SBGEMM_SMALL_K_TN
+SBGEMM_SMALL_K_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef SBGEMM_SMALL_K_TT
+SBGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)sbgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_M_PERMIT)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sbgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sbgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sbgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)sbgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
+
+ifndef SBGEMM_SMALL_K_B0_NN
+SBGEMM_SMALL_K_B0_NN = ../generic/gemm_small_matrix_kernel_nn.c
+endif
+
+ifndef SBGEMM_SMALL_K_B0_NT
+SBGEMM_SMALL_K_B0_NT = ../generic/gemm_small_matrix_kernel_nt.c
+endif
+
+ifndef SBGEMM_SMALL_K_B0_TN
+SBGEMM_SMALL_K_B0_TN = ../generic/gemm_small_matrix_kernel_tn.c
+endif
+
+ifndef SBGEMM_SMALL_K_B0_TT
+SBGEMM_SMALL_K_B0_TT = ../generic/gemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)sbgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sbgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sbgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+
+$(KDIR)sbgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DB0 $< -o $@
+endif
+
+ifndef CGEMM_SMALL_M_PERMIT
+CGEMM_SMALL_M_PERMIT = ../generic/zgemm_small_matrix_permit.c
+endif
+
+ifndef CGEMM_SMALL_K_NN
+CGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c
+endif
+
+ifndef CGEMM_SMALL_K_NT
+CGEMM_SMALL_K_NT = ../generic/zgemm_small_matrix_kernel_nt.c
+endif
+
+ifndef CGEMM_SMALL_K_TN
+CGEMM_SMALL_K_TN = ../generic/zgemm_small_matrix_kernel_tn.c
+endif
+
+ifndef CGEMM_SMALL_K_TT
+CGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)cgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_M_PERMIT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@
+
+$(KDIR)cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@
+	
+$(KDIR)cgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNR $< -o $@
+
+$(KDIR)cgemm_small_kernel_rn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRN $< -o $@
+
+$(KDIR)cgemm_small_kernel_rr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRR $< -o $@
+
+$(KDIR)cgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNT $< -o $@
+
+$(KDIR)cgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC $< -o $@
+
+$(KDIR)cgemm_small_kernel_rt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRT $< -o $@
+
+$(KDIR)cgemm_small_kernel_rc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRC=RC $< -o $@
+
+$(KDIR)cgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTN $< -o $@
+
+$(KDIR)cgemm_small_kernel_tr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTR $< -o $@
+
+$(KDIR)cgemm_small_kernel_cn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN $< -o $@
+
+$(KDIR)cgemm_small_kernel_cr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCR=CR $< -o $@
+
+$(KDIR)cgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTT $< -o $@
+
+$(KDIR)cgemm_small_kernel_tc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTC $< -o $@
+
+$(KDIR)cgemm_small_kernel_ct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCT $< -o $@
+
+$(KDIR)cgemm_small_kernel_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@
+
+ifndef CGEMM_SMALL_K_B0_NN
+CGEMM_SMALL_K_B0_NN = ../generic/zgemm_small_matrix_kernel_nn.c
+endif
+
+ifndef CGEMM_SMALL_K_B0_NT
+CGEMM_SMALL_K_B0_NT = ../generic/zgemm_small_matrix_kernel_nt.c
+endif
+
+ifndef CGEMM_SMALL_K_B0_TN
+CGEMM_SMALL_K_B0_TN = ../generic/zgemm_small_matrix_kernel_tn.c
+endif
+
+ifndef CGEMM_SMALL_K_B0_TT
+CGEMM_SMALL_K_B0_TT = ../generic/zgemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)cgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN -DB0 $< -o $@
+	
+$(KDIR)cgemm_small_kernel_b0_nr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNR -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_rn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRN -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_rr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRR -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNT -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_nc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNC -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_rt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRT -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_rc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DRC=RC -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTN -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_tr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTR -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_cn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCN -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCR=CR -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTT -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_tc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DTC -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_ct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCT -DB0 $< -o $@
+
+$(KDIR)cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC -DB0 $< -o $@
+
+ifndef ZGEMM_SMALL_M_PERMIT
+ZGEMM_SMALL_M_PERMIT = ../generic/zgemm_small_matrix_permit.c
+endif
+
+ifndef ZGEMM_SMALL_K_NN
+ZGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c
+endif
+
+ifndef ZGEMM_SMALL_K_NT
+ZGEMM_SMALL_K_NT = ../generic/zgemm_small_matrix_kernel_nt.c
+endif
+
+ifndef ZGEMM_SMALL_K_TN
+ZGEMM_SMALL_K_TN = ../generic/zgemm_small_matrix_kernel_tn.c
+endif
+
+ifndef ZGEMM_SMALL_K_TT
+ZGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)zgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_M_PERMIT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@
+
+
+$(KDIR)zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@
+	
+$(KDIR)zgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNR $< -o $@
+
+$(KDIR)zgemm_small_kernel_rn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRN $< -o $@
+
+$(KDIR)zgemm_small_kernel_rr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRR $< -o $@
+
+$(KDIR)zgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNT $< -o $@
+
+$(KDIR)zgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC $< -o $@
+
+$(KDIR)zgemm_small_kernel_rt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRT $< -o $@
+
+$(KDIR)zgemm_small_kernel_rc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRC=RC $< -o $@
+
+$(KDIR)zgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTN $< -o $@
+
+$(KDIR)zgemm_small_kernel_tr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTR $< -o $@
+
+$(KDIR)zgemm_small_kernel_cn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN $< -o $@
+
+$(KDIR)zgemm_small_kernel_cr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCR=CR $< -o $@
+
+$(KDIR)zgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTT $< -o $@
+
+$(KDIR)zgemm_small_kernel_tc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTC $< -o $@
+
+$(KDIR)zgemm_small_kernel_ct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCT $< -o $@
+
+$(KDIR)zgemm_small_kernel_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC $< -o $@
+
+ifndef ZGEMM_SMALL_K_B0_NN
+ZGEMM_SMALL_K_B0_NN = ../generic/zgemm_small_matrix_kernel_nn.c
+endif
+
+ifndef ZGEMM_SMALL_K_B0_NT
+ZGEMM_SMALL_K_B0_NT = ../generic/zgemm_small_matrix_kernel_nt.c
+endif
+
+ifndef ZGEMM_SMALL_K_B0_TN
+ZGEMM_SMALL_K_B0_TN = ../generic/zgemm_small_matrix_kernel_tn.c
+endif
+
+ifndef ZGEMM_SMALL_K_B0_TT
+ZGEMM_SMALL_K_B0_TT = ../generic/zgemm_small_matrix_kernel_tt.c
+endif
+
+$(KDIR)zgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN -DB0 $< -o $@
+	
+$(KDIR)zgemm_small_kernel_b0_nr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNR -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_rn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRN -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_rr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRR -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNT -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_nc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNC -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_rt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRT -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_rc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_NT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DRC=RC -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTN -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_tr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTR -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_cn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCN -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TN)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCR=CR -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTT -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_tc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DTC -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_ct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCT -DB0 $< -o $@
+
+$(KDIR)zgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_B0_TT)
+	$(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DCC -DB0 $< -o $@

kernel/arm64/dgemm_tcopy_8.S

@@ -50,11 +50,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define	B03		x16
 #define	B04		x17
 
-#define I		x18
-#define	J		x19
+#define I		x19
+#define	J		x20
 
-#define TEMP1		x20
-#define TEMP2		x21
+#define TEMP1		x21
 
 #define A_PREFETCH	2560
 #define B_PREFETCH	256

kernel/arm64/dtrmm_kernel_8x4.S

@@ -49,9 +49,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define pCRow3		x15
 #define pA		x16
 #define alpha		x17
-#define temp		x18
+//#define temp		x18
 #define tempOffset	x19
 #define tempK		x20
+#define temp		x21
 
 #define alpha0		d10
 #define alphaV0		v10.d[0]

kernel/arm64/sgemm_tcopy_16.S

@@ -30,7 +30,7 @@ All rights reserved.
 #define	B00		x22
 
 
-#define I		x18
+#define I		x21
 #define	J		x19
 
 #define TEMP1		x20

kernel/arm64/strmm_kernel_16x4.S

@@ -49,9 +49,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define pCRow3		x15
 #define pA		x16
 #define alpha		w17
-#define temp		x18
+//#define temp		x18
 #define tempOffset	x19
 #define tempK		x20
+#define temp            x21
 
 #define alpha0		s10
 #define alphaV0		v10.s[0]

kernel/arm64/zgemm_kernel_4x4.S

@@ -48,8 +48,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define pCRow2		x14
 #define pCRow3		x15
 #define pA		x16
-#define alphaR		x17
-#define alphaI		x18
+#define alphaR		x19
+#define alphaI		x20
 
 #define alpha0_R	d10
 #define alphaV0_R	v10.d[0]

kernel/arm64/ztrmm_kernel_4x4.S

@@ -49,7 +49,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define pCRow3		x15
 #define pA		x16
 #define alphaR		x17
-#define alphaI		x18
+#define alphaI		x22
 #define temp		x19
 #define tempOffset	x20
 #define tempK		x21

kernel/generic/dot.c

@@ -47,7 +47,6 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 
 	if ( (inc_x == 1) && (inc_y == 1) )
 	{
-        int n1 = n & -4;
 #if V_SIMD && !defined(DSDOT)
         const int vstep = v_nlanes_f32;
         const int unrollx4 = n & (-vstep * 4);
@@ -84,6 +83,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
         }
         dot = v_sum_f32(vsum0);
 #elif defined(DSDOT)
+        int n1 = n & -4;
 		for (; i < n1; i += 4)
 		{
 			dot += (double) y[i] * (double) x[i]
@@ -92,6 +92,7 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 			    + (double) y[i+3] * (double) x[i+3] ;
 		}
 #else
+        int n1 = n & -4;
 		for (; i < n1; i += 4)
 		{
 			dot += y[i] * x[i]

kernel/generic/gemm_small_matrix_kernel_nn.c

@@ -0,0 +1,56 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifdef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	//naive implemtation
+	//Column major
+
+	BLASLONG i,j,k;
+	FLOAT result=0.0;
+
+	for(i=0; i<M; i++){
+		for(j=0; j<N; j++){
+			result=0.0;
+			for(k=0; k<K; k++){
+				result += A[i+k*lda] * B[k+j*ldb];
+			}
+#ifdef B0
+			C[i+j*ldc]=alpha * result;
+#else
+			C[i+j*ldc]=C[i+j*ldc] * beta + alpha * result;
+#endif
+		}
+	}
+	return 0;
+}

kernel/generic/gemm_small_matrix_kernel_nt.c

@@ -0,0 +1,56 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifdef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	//naive implemtation
+	//Column major
+
+	BLASLONG i,j,k;
+	FLOAT result=0.0;
+
+	for(i=0; i<M; i++){
+		for(j=0; j<N; j++){
+			result=0.0;
+			for(k=0; k<K; k++){
+				result += A[i+k*lda] * B[k*ldb+j];
+			}
+#ifdef B0
+			C[i+j*ldc]=alpha * result;
+#else
+			C[i+j*ldc]=C[i+j*ldc] * beta + alpha * result;
+#endif
+		}
+	}
+	return 0;
+}

kernel/generic/gemm_small_matrix_kernel_tn.c

@@ -0,0 +1,57 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifdef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	//naive implemtation
+	//Column major
+
+	BLASLONG i,j,k;
+	FLOAT result=0.0;
+
+	for(i=0; i<M; i++){
+		for(j=0; j<N; j++){
+			result=0.0;
+			for(k=0; k<K; k++){
+				result += A[i*lda+k] * B[k+j*ldb];
+			}
+#ifdef B0
+			C[i+j*ldc]=alpha * result;
+#else
+			C[i+j*ldc]=C[i+j*ldc] * beta + alpha * result;
+#endif
+		}
+	}
+
+	return 0;
+}

kernel/generic/gemm_small_matrix_kernel_tt.c

@@ -0,0 +1,57 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifdef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	//naive implemtation
+	//Column major
+
+	BLASLONG i,j,k;
+	FLOAT result=0.0;
+
+	for(i=0; i<M; i++){
+		for(j=0; j<N; j++){
+			result=0.0;
+			for(k=0; k<K; k++){
+				result += A[i*lda+k] * B[k*ldb+j];
+			}
+#ifdef B0
+			C[i+j*ldc]=alpha * result;
+#else
+			C[i+j*ldc]=C[i+j*ldc] * beta + alpha * result;
+#endif
+		}
+	}
+
+	return 0;
+}

kernel/generic/gemm_small_matrix_permit.c

@@ -0,0 +1,40 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta)
+{
+	return 0;
+/*
+	double MNK = (double) M * (double) N * (double) K;
+	if (MNK <= 100.0*100.0*100.0)
+		return 1;
+	else
+		return 0;
+*/
+}

kernel/generic/zgemm_small_matrix_kernel_nn.c

@@ -0,0 +1,89 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifndef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT beta0, FLOAT beta1, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	FLOAT real, imag;
+#ifndef B0
+	FLOAT tmp0, tmp1;
+#endif
+	int i, j, l;
+	for(i = 0; i < M; i++){
+		for(j = 0; j < N; j++){
+			real=0;
+			imag=0;
+
+			for(l = 0; l < K; l++){
+#if defined(NN)
+				real += (A[l*2*lda + 2*i]*B[j*2*ldb + 2*l]      
+					 -A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(A[l*2*lda + 2*i] * B[j*2*ldb + 2*l + 1]        
+				       + A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l]);
+#elif defined(NR)
+				real += (A[l*2*lda + 2*i]*B[j*2*ldb + 2*l]      
+					 +A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(-A[l*2*lda + 2*i] * B[j*2*ldb + 2*l + 1]      
+				       + A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l]);
+#elif defined(RN)
+				real += (A[l*2*lda + 2*i]*B[j*2*ldb + 2*l]
+					 +A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(A[l*2*lda + 2*i] * B[j*2*ldb + 2*l + 1]
+				       - A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l]);
+#elif defined(RR)
+				real += (A[l*2*lda + 2*i]*B[j*2*ldb + 2*l]
+					 -A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(-A[l*2*lda + 2*i] * B[j*2*ldb + 2*l + 1]
+				       - A[l*2*lda + 2*i + 1] * B[j*2*ldb + 2*l]);
+#endif
+			}
+
+#ifndef B0
+			tmp0 = beta0*C[j*2*ldc + 2*i] - beta1*C[j*2*ldc+ 2*i + 1];
+			tmp1 = beta0*C[j*2*ldc+ 2*i + 1] + beta1*C[j*2*ldc + 2*i];
+
+
+			C[j*2*ldc + 2*i] =tmp0+ alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = tmp1+ alpha0*imag + real*alpha1;
+#else
+			C[j*2*ldc + 2*i] = alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = alpha0*imag + real*alpha1;
+#endif
+		}
+	}
+	
+	return 0;
+}

kernel/generic/zgemm_small_matrix_kernel_nt.c

@@ -0,0 +1,93 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifndef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT beta0, FLOAT beta1, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	FLOAT real, imag;
+#ifndef B0
+	FLOAT tmp0, tmp1;
+#endif
+	int i, j, l;
+	for(i = 0; i < M; i++){
+		for(j = 0; j < N; j++){
+			real=0;
+			imag=0;
+
+			for(l = 0; l < K; l++){
+#if defined(NT)
+				real += (A[l*2*lda + 2*i]*B[l*2*ldb + 2*j]
+					 -A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(A[l*2*lda + 2*i] * B[l*2*ldb + 2*j + 1]    
+				       + A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(NC)
+		                real += (A[l*2*lda + 2*i]*B[l*2*ldb + 2*j]          
+					 +A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(-A[l*2*lda + 2*i] * B[l*2*ldb + 2*j + 1]
+				       + A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(RT)
+		                real += (A[l*2*lda + 2*i]*B[l*2*ldb + 2*j]
+					 +A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(A[l*2*lda + 2*i] * B[l*2*ldb + 2*j + 1]    
+				       - A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(RC)
+		                real += (A[l*2*lda + 2*i]*B[l*2*ldb + 2*j]
+					 -A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(-A[l*2*lda + 2*i] * B[l*2*ldb + 2*j + 1]
+				       - A[l*2*lda + 2*i + 1] * B[l*2*ldb + 2*j]);
+
+#endif
+			}
+
+#ifndef B0
+			tmp0 = beta0*C[j*2*ldc + 2*i] - beta1*C[j*2*ldc+ 2*i + 1];
+			tmp1 = beta0*C[j*2*ldc+ 2*i + 1] + beta1*C[j*2*ldc + 2*i];
+
+
+			C[j*2*ldc + 2*i] =tmp0+ alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = tmp1+ alpha0*imag + real*alpha1;
+#else
+			C[j*2*ldc + 2*i] = alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = alpha0*imag + real*alpha1;
+#endif
+		}
+	}
+	
+	return 0;
+}

kernel/generic/zgemm_small_matrix_kernel_tn.c

@@ -0,0 +1,93 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifndef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT beta0, FLOAT beta1, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	FLOAT real, imag;
+#ifndef B0
+	FLOAT tmp0, tmp1;
+#endif
+	int i, j, l;
+	for(i = 0; i < M; i++){
+		for(j = 0; j < N; j++){
+			real=0;
+			imag=0;
+
+			for(l = 0; l < K; l++){
+#if defined(TN)
+				real += (A[i*2*lda + 2*l]*B[j*2*ldb + 2*l]
+					 -A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(A[i*2*lda + 2*l] * B[j*2*ldb + 2*l + 1]
+				       + A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l]);
+
+#elif defined(TR)
+				real += (A[i*2*lda + 2*l]*B[j*2*ldb + 2*l]
+					 +A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(-A[i*2*lda + 2*l] * B[j*2*ldb + 2*l + 1]
+				       + A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l]);
+
+#elif defined(CN)
+				real += (A[i*2*lda + 2*l]*B[j*2*ldb + 2*l]
+					 +A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(A[i*2*lda + 2*l] * B[j*2*ldb + 2*l + 1]
+				       - A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l]);
+
+#elif defined(CR)
+				real += (A[i*2*lda + 2*l]*B[j*2*ldb + 2*l]
+					 -A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l + 1]);
+
+				imag+=(-A[i*2*lda + 2*l] * B[j*2*ldb + 2*l + 1]
+				       - A[i*2*lda + 2*l + 1] * B[j*2*ldb + 2*l]);
+
+#endif
+			}
+
+#ifndef B0
+			tmp0 = beta0*C[j*2*ldc + 2*i] - beta1*C[j*2*ldc+ 2*i + 1];
+			tmp1 = beta0*C[j*2*ldc+ 2*i + 1] + beta1*C[j*2*ldc + 2*i];
+
+
+			C[j*2*ldc + 2*i] =tmp0+ alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = tmp1+ alpha0*imag + real*alpha1;
+#else
+			C[j*2*ldc + 2*i] = alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = alpha0*imag + real*alpha1;
+#endif
+		}
+	}
+	
+	return 0;
+}

kernel/generic/zgemm_small_matrix_kernel_tt.c

@@ -0,0 +1,93 @@
+/***************************************************************************
+Copyright (c) 2020, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+#ifndef B0
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT beta0, FLOAT beta1, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha0, FLOAT alpha1, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	FLOAT real, imag;
+#ifndef B0
+	FLOAT tmp0, tmp1;
+#endif
+	int i, j, l;
+	for(i = 0; i < M; i++){
+		for(j = 0; j < N; j++){
+			real=0;
+			imag=0;
+
+			for(l = 0; l < K; l++){
+#if defined(TT)
+				real += (A[i*2*lda + 2*l]*B[l*2*ldb + 2*j]
+					 -A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(A[i*2*lda + 2*l] * B[l*2*ldb + 2*j + 1]
+				       + A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(TC)
+				real += (A[i*2*lda + 2*l]*B[l*2*ldb + 2*j]
+					 +A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(-A[i*2*lda + 2*l] * B[l*2*ldb + 2*j + 1]
+				       + A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(CT)
+				real += (A[i*2*lda + 2*l]*B[l*2*ldb + 2*j]
+					 +A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(A[i*2*lda + 2*l] * B[l*2*ldb + 2*j + 1]
+				       - A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j]);
+
+#elif defined(CC)
+				real += (A[i*2*lda + 2*l]*B[l*2*ldb + 2*j]
+					 -A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j + 1]);
+
+				imag+=(-A[i*2*lda + 2*l] * B[l*2*ldb + 2*j + 1]
+				       - A[i*2*lda + 2*l + 1] * B[l*2*ldb + 2*j]);
+
+#endif
+			}
+
+#ifndef B0
+			tmp0 = beta0*C[j*2*ldc + 2*i] - beta1*C[j*2*ldc+ 2*i + 1];
+			tmp1 = beta0*C[j*2*ldc+ 2*i + 1] + beta1*C[j*2*ldc + 2*i];
+
+
+			C[j*2*ldc + 2*i] =tmp0+ alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = tmp1+ alpha0*imag + real*alpha1;
+#else
+			C[j*2*ldc + 2*i] = alpha0*real - alpha1*imag;
+			C[j*2*ldc+ 2*i + 1] = alpha0*imag + real*alpha1;
+#endif
+		}
+	}
+	
+	return 0;
+}

kernel/generic/zgemm_small_matrix_permit.c

@@ -0,0 +1,40 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha0, FLOAT alpha1, FLOAT beta0, FLOAT beta1)
+{
+	return 0;
+/*
+	double MNK = (double) M * (double) N * (double) K;
+	if (MNK <= 100.0*100.0*100.0)
+		return 1;
+	else
+		return 0;
+*/
+}

kernel/loongarch64/KERNEL

@@ -234,3 +234,5 @@ endif
 ifndef ZGEMM3MKERNEL
 ZGEMM3MKERNEL    =  zgemm3m_kernel.S
 endif
+
+DSDOTKERNEL  = dot.S

kernel/loongarch64/asum.S

@@ -54,7 +54,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    MTC  s1, $r0
    MTC  s2, $r0
    slli.d  INCX, INCX, BASE_SHIFT
-   li TEMP, SIZE
+   li.d TEMP, SIZE
    bge $r0,    N, .L999
    srai.d  I, N, 3
    bne INCX, TEMP, .L20

kernel/loongarch64/cnrm2.S

@@ -57,7 +57,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
    movgr2fr.d  s1,  $r0
-   li  TEMP, 2 * SIZE
+   li.d  TEMP, 2 * SIZE
    fmov.d s2, s1
    bge $r0,    N, .L999
    slli.d INCX, INCX, ZBASE_SHIFT

kernel/loongarch64/copy.S

@@ -52,7 +52,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    LDINT   INCY,  0(INCY)
 #endif
 
-   li  TEMP, SIZE
+   li.d  TEMP, SIZE
    NOP
    slli.d INCX, INCX, BASE_SHIFT
    bge $r0,    N, .L999

kernel/loongarch64/dot.S

@@ -57,7 +57,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    MTC  s1, $r0
    MTC  s2, $r0
    slli.d  INCX, INCX, BASE_SHIFT
-   li  TEMP, SIZE
+   li.d  TEMP, SIZE
    slli.d INCY, INCY, BASE_SHIFT
    bge $r0,    N, .L999
    srai.d I, N, 3

kernel/loongarch64/gemv_n.S

@@ -91,7 +91,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    bge $r0,    M, .L999
    slli.d INCY, INCY, BASE_SHIFT
    bge $r0,    N, .L999
-   li  I, SIZE
+   li.d  I, SIZE
    move   YORIG, Y
    beq INCY, I, .L10
    srai.d  I,  M, 2
@@ -472,7 +472,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    .align 3
 
 .L900:
-   li  YORIG, SIZE
+   li.d  YORIG, SIZE
    srai.d I,  M, 2
    beq INCY, YORIG, .L999
    move   XX, BUFFER

kernel/loongarch64/gemv_t.S

@@ -88,7 +88,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    bge $r0,    M, .L999
    slli.d INCY, INCY, BASE_SHIFT
    bge $r0,    N, .L999
-   li  I, SIZE
+   li.d  I, SIZE
    move   XORIG, X
    beq INCX, I, .L10
    srai.d  I,  M, 2

kernel/loongarch64/iamax.S

@@ -62,24 +62,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    LDINT   INCX,  0(INCX)
 #endif
 
-   li x1, 0
+   li.d x1, 0
    bge $r0,    N, .L999
    slli.d INCX, INCX, BASE_SHIFT
    bge $r0,    INCX, .L999
    LD a1,  X,   0 * SIZE
    addi.d  N, N, -1
-   li x1, 1
+   li.d x1, 1
    bge $r0,    N, .L999
    FABS    s1, a1
    add.d   X, X, INCX
    FABS    s2, a1
-   li  x2, 1
+   li.d  x2, 1
    FABS    s3, a1
    srai.d  I, N, 3
    FABS    s4, a1
-   li  x3, 1
-   li  TEMP, 2
-   li x4, 1
+   li.d  x3, 1
+   li.d  TEMP, 2
+   li.d x4, 1
    bge $r0,    I, .L15
    LD a1,  X,   0 * SIZE
    add.d   X, X, INCX

kernel/loongarch64/iamin.S

@@ -62,24 +62,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    LDINT   INCX,  0(INCX)
 #endif
 
-   li x1, 0
+   li.d x1, 0
    bge $r0,    N, .L999
    slli.d INCX, INCX, BASE_SHIFT
    bge $r0,    INCX, .L999
    LD a1,  X,   0 * SIZE
    addi.d  N, N, -1
-   li x1, 1
+   li.d x1, 1
    bge $r0,    N, .L999
    FABS    s1, a1
    add.d   X, X, INCX
    FABS    s2, a1
-   li  x2, 1
+   li.d  x2, 1
    FABS    s3, a1
    srai.d  I, N, 3
    FABS    s4, a1
-   li  x3, 1
-   li  TEMP, 2
-   li x4, 1
+   li.d  x3, 1
+   li.d  TEMP, 2
+   li.d x4, 1
    bge $r0,    I, .L15
    LD a1,  X,   0 * SIZE
    add.d   X, X, INCX

kernel/loongarch64/izamax.S

@@ -66,7 +66,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       LDINT   INCX,  0(INCX)
 #endif
 
-   li x1, 0
+   li.d x1, 0
    bge $r0,    N, .L999
    slli.d INCX, INCX, ZBASE_SHIFT
    bge $r0,    INCX, .L999
@@ -79,14 +79,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    ADD s3, t1, t2
    ADD s4, t1, t2
    addi.d  N, N, -1
-   li x1, 1
+   li.d x1, 1
    bge $r0,    N, .L999
    add.d   X, X, INCX
-   li  x2, 1
+   li.d  x2, 1
    srai.d  I, N, 2
-   li  x3, 1
-   li  TEMP, 2
-   li x4, 1
+   li.d  x3, 1
+   li.d  TEMP, 2
+   li.d x4, 1
    bge $r0,    I, .L15
    LD a1,  X,   0 * SIZE
    LD a2,  X,   1 * SIZE

kernel/loongarch64/izamin.S

@@ -66,7 +66,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    LDINT   INCX,  0(INCX)
 #endif
 
-   li x1, 0
+   li.d x1, 0
    bge $r0,    N, .L999
    slli.d INCX, INCX, ZBASE_SHIFT
    bge $r0,    INCX, .L999
@@ -79,14 +79,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    ADD s3, t1, t2
    ADD s4, t1, t2
    addi.d  N, N, -1
-   li x1, 1
+   li.d x1, 1
    bge $r0,    N, .L999
    add.d   X, X, INCX
-   li  x2, 1
+   li.d  x2, 1
    srai.d  I, N, 2
-   li  x3, 1
-   li  TEMP, 2
-   li x4, 1
+   li.d  x3, 1
+   li.d  TEMP, 2
+   li.d x4, 1
    bge $r0,    I, .L15
    LD a1,  X,   0 * SIZE
    LD a2,  X,   1 * SIZE

kernel/loongarch64/scal.S

@@ -52,7 +52,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    PROLOGUE
 
-   li  TEMP, SIZE
+   li.d  TEMP, SIZE
    MTC  a1, $r0
    slli.d INCX, INCX, BASE_SHIFT
    bge $r0,    N, .L999

kernel/loongarch64/snrm2.S

@@ -57,7 +57,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
    movgr2fr.d  s1, $r0
-   li  TEMP, SIZE
+   li.d  TEMP, SIZE
    fmov.d s2, s1
    bge $r0,    N, .L999
    slli.d INCX, INCX, BASE_SHIFT

kernel/loongarch64/swap.S

@@ -58,7 +58,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    PROLOGUE
 
-   li  TEMP, SIZE
+   li.d  TEMP, SIZE
    slli.d INCX, INCX, BASE_SHIFT
    bge $r0,    N, .L999
    slli.d INCY, INCY, BASE_SHIFT

kernel/loongarch64/zcopy.S

@@ -53,7 +53,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    LDINT   INCY,  0(INCY)
 #endif
 
-   li  TEMP, 2 * SIZE
+   li.d  TEMP, 2 * SIZE
    NOP
    slli.d INCX, INCX, ZBASE_SHIFT
    bge $r0,    N, .L999

kernel/loongarch64/zdot.S

@@ -62,7 +62,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    MOV s3, s2
    MOV s4, s3
    slli.d  INCX, INCX, ZBASE_SHIFT
-   li  TEMP, 2 * SIZE
+   li.d  TEMP, 2 * SIZE
    slli.d INCY, INCY, ZBASE_SHIFT
    bge $r0,    N, .L999
    srai.d I, N, 2

kernel/loongarch64/zgemv_n.S

@@ -123,7 +123,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    bge $r0,    M, .L999
    slli.d INCY, INCY, ZBASE_SHIFT
    bge $r0,    N, .L999
-   li  I, 2 * SIZE
+   li.d  I, 2 * SIZE
    move   YORIG, Y
    beq INCY, I, .L10
    srai.d  I,  M, 2
@@ -576,7 +576,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    .align 3
 
 .L900:
-   li  YORIG, 2 * SIZE
+   li.d  YORIG, 2 * SIZE
    srai.d I,  M, 2
    beq INCY, YORIG, .L999
    move   XX, BUFFER

kernel/loongarch64/zgemv_t.S

@@ -116,7 +116,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    bge $r0,    M, .L999
    slli.d INCY, INCY, ZBASE_SHIFT
    bge $r0,    N, .L999
-   li  I, 2 * SIZE
+   li.d  I, 2 * SIZE
    move   XORIG, X
    beq INCX, I, .L10
    srai.d  I,  M, 2

kernel/loongarch64/zscal.S

@@ -52,7 +52,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    PROLOGUE
 
-   li  TEMP, 2 * SIZE
+   li.d  TEMP, 2 * SIZE
    MTC  a1, $r0
    slli.d INCX, INCX, ZBASE_SHIFT
    bge $r0,    N, .L999

kernel/power/dasum.c

@@ -115,14 +115,14 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
 	{
 
 #if defined(POWER10) && (__BYTE_ORDER__ != __ORDER_BIG_ENDIAN__)
-		if ( n >= 16 )
+		if ( n >= 32)
 		{
 			BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3;
 			for (i = 0; i < align; i++) {
 				sumf += ABS(x[i]);
 			}
 		}
-		n1 = (n-i) & -16;
+		n1 = (n-i) & -32;
 		if ( n1 > 0 )
 		{
 			sumf += dasum_kernel_16(n1, &x[i]);

kernel/power/dasum_microk_power10.c

@@ -34,6 +34,19 @@ static double dasum_kernel_16 (long n, double *x)
   __vector double t1;
   __vector double t2;
   __vector double t3;
+  __vector double t4;
+  __vector double t5;
+  __vector double t6;
+  __vector double t7;
+  __vector double a0;
+  __vector double a1;
+  __vector double a2;
+  __vector double a3;
+  __vector double a4;
+  __vector double a5;
+  __vector double a6;
+  __vector double a7;
+
 
   __asm__
     (
@@ -48,14 +61,27 @@ static double dasum_kernel_16 (long n, double *x)
        "xxlxor		38, 38,	38	\n\t"
        "xxlxor		39, 39,	39	\n\t"
 
+       "xxlxor		%x11, %x11, %x11	\n\t"
+       "xxlxor		%x12, %x12, %x12	\n\t"
+       "xxlxor		%x13, %x13, %x13	\n\t"
+       "xxlxor		%x14, %x14, %x14 	\n\t"
+       "xxlxor		%x15, %x15, %x15 	\n\t"
+       "xxlxor		%x16, %x16, %x16	\n\t"
+       "xxlxor		%x17, %x17, %x17	\n\t"
+       "xxlxor		%x18, %x18, %x18	\n\t"
+
        "lxvp            40, 0(%2)       \n\t"
        "lxvp            42, 32(%2)      \n\t"
        "lxvp            44, 64(%2)      \n\t"
        "lxvp            46, 96(%2)      \n\t"
+       "lxvp            52, 128(%2)	\n\t"
+       "lxvp            54, 160(%2)	\n\t"
+       "lxvp            56, 192(%2)	\n\t"
+       "lxvp            58, 224(%2)	\n\t"
 
-       "addi		%2, %2, 128	\n\t"
+       "addi		%2, %2, 256	\n\t"
 
-       "addic.		%1, %1, -16	\n\t"
+       "addic.		%1, %1, -32	\n\t"
        "ble		two%=		\n\t"
 
        ".align	5		\n"
@@ -65,33 +91,52 @@ static double dasum_kernel_16 (long n, double *x)
        "xvabsdp		49, 41		\n\t"
        "xvabsdp		50, 42		\n\t"
        "xvabsdp		51, 43		\n\t"
-       "lxvp            40, 0(%2)       \n\t"
-
 
        "xvabsdp		%x3, 44		\n\t"
        "xvabsdp		%x4, 45		\n\t"
-       "lxvp            42, 32(%2)      \n\t"
-
-
        "xvabsdp		%x5, 46		\n\t"
        "xvabsdp		%x6, 47		\n\t"
-       "lxvp            44, 64(%2)      \n\t"
-
 
        "xvadddp		32, 32, 48	\n\t"
        "xvadddp		33, 33, 49	\n\t"
-
-       "lxvp            46, 96(%2)      \n\t"
-
        "xvadddp		34, 34, 50	\n\t"
        "xvadddp		35, 35, 51	\n\t"
-       "addi		%2, %2, 128	\n\t"
+       "lxvp            40, 0(%2)       \n\t"
+       "lxvp            42, 32(%2)      \n\t"
+       "lxvp            44, 64(%2)      \n\t"
+       "lxvp            46, 96(%2)      \n\t"
+
        "xvadddp		36, 36, %x3	\n\t"
        "xvadddp		37, 37, %x4	\n\t"
-       "addic.		%1, %1, -16	\n\t"
        "xvadddp		38, 38, %x5	\n\t"
        "xvadddp		39, 39, %x6	\n\t"
 
+       "xvabsdp		60, 52 		\n\t"
+       "xvabsdp		61, 53 		\n\t"
+       "xvabsdp		62, 54 		\n\t"
+       "xvabsdp		63, 55 		\n\t"
+
+       "xvabsdp		%x7, 56		\n\t"
+       "xvabsdp		%x8, 57		\n\t"
+       "xvabsdp		%x9, 58		\n\t"
+       "xvabsdp		%x10, 59	\n\t"
+
+       "xvadddp		%x11, %x11, 60	\n\t"
+       "xvadddp		%x12, %x12, 61	\n\t"
+       "xvadddp		%x13, %x13, 62	\n\t"
+       "xvadddp		%x14, %x14, 63	\n\t"
+
+       "lxvp		52, 128(%2)	\n\t"
+       "lxvp		54, 160(%2)	\n\t"
+       "lxvp		56, 192(%2)	\n\t"
+       "lxvp		58, 224(%2)	\n\t"
+       "xvadddp		%x15, %x15, %x7	\n\t"
+       "xvadddp		%x16, %x16, %x8	\n\t"
+       "xvadddp		%x17, %x17, %x9	\n\t"
+       "xvadddp		%x18, %x18, %x10	\n\t"
+       "addi		%2, %2, 256	\n\t"
+       "addic.		%1, %1, -32	\n\t"
+
        "bgt		one%=		\n"
 
      "two%=:				\n\t"
@@ -114,6 +159,25 @@ static double dasum_kernel_16 (long n, double *x)
        "xvadddp		38, 38, %x5	\n\t"
        "xvadddp		39, 39, %x6	\n\t"
 
+       "xvabsdp		60, 52 		\n\t"
+       "xvabsdp		61, 53 		\n\t"
+       "xvabsdp		62, 54 		\n\t"
+       "xvabsdp		63, 55 		\n\t"
+
+       "xvabsdp		%x7, 56		\n\t"
+       "xvabsdp		%x8, 57		\n\t"
+       "xvabsdp		%x9, 58 	\n\t"
+       "xvabsdp		%x10, 59	\n\t"
+       "xvadddp		%x11, %x11, 60	\n\t"
+       "xvadddp		%x12, %x12, 61	\n\t"
+       "xvadddp		%x13, %x13, 62	\n\t"
+       "xvadddp		%x14, %x14, 63	\n\t"
+
+       "xvadddp		%x15, %x15, %x7	\n\t"
+       "xvadddp		%x16, %x16, %x8	\n\t"
+       "xvadddp		%x17, %x17, %x9	\n\t"
+       "xvadddp		%x18, %x18, %x10	\n\t"
+
        "xvadddp		32, 32, 33	\n\t"
        "xvadddp		34, 34, 35	\n\t"
        "xvadddp		36, 36, 37	\n\t"
@@ -122,7 +186,18 @@ static double dasum_kernel_16 (long n, double *x)
        "xvadddp		32, 32, 34	\n\t"
        "xvadddp		36, 36, 38	\n\t"
 
+       "xvadddp		%x11, %x11, %x12 	\n\t"
+       "xvadddp		%x13, %x13, %x14	\n\t"
+       "xvadddp		%x15, %x15, %x16	\n\t"
+       "xvadddp		%x17, %x17, %x18	\n\t"
+
+       "xvadddp		%x11, %x11, %x13	\n\t"
+       "xvadddp		%x15, %x15, %x17	\n\t"
+
+       "xvadddp		%x11, %x11, %x15	\n\t"
+
        "xvadddp		32, 32, 36	\n\t"
+       "xvadddp		32, 32, %x11	\n\t"
 
        XXSWAPD_S(33,32)
        "xsadddp		%x0, 32, 33	\n"
@@ -136,14 +211,27 @@ static double dasum_kernel_16 (long n, double *x)
        "=wa" (t0),	// 3
        "=wa" (t1),	// 4
        "=wa" (t2),	// 5
-       "=wa" (t3)	// 6
+       "=wa" (t3),	// 6
+       "=wa" (t4),	// 7
+       "=wa" (t5),	// 8
+       "=wa" (t6),	// 9
+       "=wa" (t7),	// 10
+       "=wa" (a0),	// 11
+       "=wa" (a1),	// 12
+       "=wa" (a2),	// 13
+       "=wa" (a3),	// 14
+       "=wa" (a4),	// 15
+       "=wa" (a5),	// 16
+       "=wa" (a6),	// 17
+       "=wa" (a7)	// 18
      :
        "m" (*x)
      :
        "cr0",
        "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
        "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47",
-       "vs48","vs49","vs50","vs51"
+       "vs48","vs49","vs50","vs51","vs52","vs53","vs54","vs55",
+       "vs56","vs57","vs58","vs59","vs60","vs61","vs62","vs63"
      );
 
   return sum;

kernel/power/drot.c

@@ -110,8 +110,6 @@ int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT
 {
 	BLASLONG i=0;
 	BLASLONG ix=0,iy=0;
-	FLOAT *x1=x;
-	FLOAT *y1=y;
 	FLOAT temp;
 
 	if ( n <= 0     )  return(0);
@@ -139,7 +137,7 @@ int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT
 		BLASLONG n1 = n & -16;
 		if ( n1 > 0 )
 		{
-			drot_kernel_16(n1, x1, y1, c, s);
+			drot_kernel_16(n1, x, y, c, s);
 			i=n1;
 		}
 #endif

kernel/power/idamax.c

@@ -330,10 +330,10 @@ BLASLONG CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x) {
 
     if (inc_x == 1) {
 
-        BLASLONG n1 = n & -32;
 #if defined(_CALL_ELF) && (_CALL_ELF == 2)
 #if defined(__VEC__) || defined(__ALTIVEC__)
 
+        BLASLONG n1 = n & -32;
 	if (n1 > 0) {
 
             max = diamax_kernel_32(n1, x, &maxf);

kernel/power/trsm_kernel_LN_power10.c

@@ -389,7 +389,6 @@ static inline __attribute__ ((always_inline)) void solve16x8(FLOAT *a, FLOAT *b,
    vector FLOAT *Vc6 = (vector FLOAT *) c6;
    vector FLOAT *Vc7 = (vector FLOAT *) c7;
    vector FLOAT VbS0, VbS1, VbS2, VbS3, VbS4, VbS5, VbS6, VbS7;
-   int  j;
 
    b[120] = (c0[15] *= a[255]);
    b[121] = (c1[15] *= a[255]);

kernel/power/trsm_kernel_LT_power10.c

@@ -390,7 +390,6 @@ static inline __attribute__ ((always_inline)) void solve16x8(FLOAT *a, FLOAT *b,
    vector FLOAT *Vc6 = (vector FLOAT *) c6;
    vector FLOAT *Vc7 = (vector FLOAT *) c7;
    vector FLOAT VbS0, VbS1, VbS2, VbS3, VbS4, VbS5, VbS6, VbS7;
-   int  j;
 
    b[0] = (c0[0] *= a[0]);
    b[1] = (c1[0] *= a[0]);

kernel/power/zgemv_n_4.c

@@ -607,7 +607,6 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT * buffer) {
     BLASLONG i;
-    BLASLONG j;
     FLOAT *a_ptr;
     FLOAT *x_ptr;
     FLOAT *y_ptr;

kernel/power/zgemv_n_power10.c

@@ -738,7 +738,6 @@ static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest, FLOAT
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha_r, FLOAT alpha_i, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT * buffer) {
     BLASLONG i;
-    BLASLONG j;
     FLOAT *a_ptr;
     FLOAT *x_ptr;
     FLOAT *y_ptr;

kernel/setparam-ref.c

@@ -112,6 +112,11 @@ gotoblas_t TABLE_NAME = {
 #else
   NULL,NULL,
 #endif
+#ifdef SMALL_MATRIX_OPT
+  sbgemm_small_matrix_permitTS,
+  sbgemm_small_kernel_nnTS, sbgemm_small_kernel_ntTS, sbgemm_small_kernel_tnTS, sbgemm_small_kernel_ttTS,
+  sbgemm_small_kernel_b0_nnTS, sbgemm_small_kernel_b0_ntTS, sbgemm_small_kernel_b0_tnTS, sbgemm_small_kernel_b0_ttTS,
+#endif
 #endif
 
 #if ( BUILD_SINGLE==1) || (BUILD_DOUBLE==1) || (BUILD_COMPLEX==1) || (BUILD_COMPLEX16==1)
@@ -171,6 +176,14 @@ gotoblas_t TABLE_NAME = {
   sgemm_oncopyTS, sgemm_otcopyTS,
 #endif
 
+#if BUILD_SINGLE == 1
+#ifdef SMALL_MATRIX_OPT
+  sgemm_small_matrix_permitTS,
+  sgemm_small_kernel_nnTS, sgemm_small_kernel_ntTS, sgemm_small_kernel_tnTS, sgemm_small_kernel_ttTS,
+  sgemm_small_kernel_b0_nnTS, sgemm_small_kernel_b0_ntTS, sgemm_small_kernel_b0_tnTS, sgemm_small_kernel_b0_ttTS,
+#endif
+#endif
+
 #if (BUILD_SINGLE==1) || (BUILD_DOUBLE==1) 
   strsm_kernel_LNTS, strsm_kernel_LTTS, strsm_kernel_RNTS, strsm_kernel_RTTS,
 #if SGEMM_DEFAULT_UNROLL_M != SGEMM_DEFAULT_UNROLL_N
@@ -257,6 +270,11 @@ gotoblas_t TABLE_NAME = {
 #endif
 
 #if  (BUILD_DOUBLE==1)  
+#ifdef SMALL_MATRIX_OPT
+  dgemm_small_matrix_permitTS,
+  dgemm_small_kernel_nnTS, dgemm_small_kernel_ntTS, dgemm_small_kernel_tnTS, dgemm_small_kernel_ttTS,
+  dgemm_small_kernel_b0_nnTS, dgemm_small_kernel_b0_ntTS, dgemm_small_kernel_b0_tnTS, dgemm_small_kernel_b0_ttTS,
+#endif
   dtrsm_kernel_LNTS, dtrsm_kernel_LTTS, dtrsm_kernel_RNTS, dtrsm_kernel_RTTS,
 #if DGEMM_DEFAULT_UNROLL_M != DGEMM_DEFAULT_UNROLL_N
   dtrsm_iunucopyTS, dtrsm_iunncopyTS, dtrsm_iutucopyTS, dtrsm_iutncopyTS,
@@ -389,6 +407,18 @@ gotoblas_t TABLE_NAME = {
 #endif
   cgemm_oncopyTS, cgemm_otcopyTS,
 
+#ifdef SMALL_MATRIX_OPT
+  cgemm_small_matrix_permitTS,
+  cgemm_small_kernel_nnTS, cgemm_small_kernel_ntTS, cgemm_small_kernel_nrTS, cgemm_small_kernel_ncTS,
+  cgemm_small_kernel_tnTS, cgemm_small_kernel_ttTS, cgemm_small_kernel_trTS, cgemm_small_kernel_tcTS,
+  cgemm_small_kernel_rnTS, cgemm_small_kernel_rtTS, cgemm_small_kernel_rrTS, cgemm_small_kernel_rcTS,
+  cgemm_small_kernel_cnTS, cgemm_small_kernel_ctTS, cgemm_small_kernel_crTS, cgemm_small_kernel_ccTS,
+  cgemm_small_kernel_b0_nnTS, cgemm_small_kernel_b0_ntTS, cgemm_small_kernel_b0_nrTS, cgemm_small_kernel_b0_ncTS,
+  cgemm_small_kernel_b0_tnTS, cgemm_small_kernel_b0_ttTS, cgemm_small_kernel_b0_trTS, cgemm_small_kernel_b0_tcTS,
+  cgemm_small_kernel_b0_rnTS, cgemm_small_kernel_b0_rtTS, cgemm_small_kernel_b0_rrTS, cgemm_small_kernel_b0_rcTS,
+  cgemm_small_kernel_b0_cnTS, cgemm_small_kernel_b0_ctTS, cgemm_small_kernel_b0_crTS, cgemm_small_kernel_b0_ccTS,
+#endif
+
   ctrsm_kernel_LNTS, ctrsm_kernel_LTTS, ctrsm_kernel_LRTS, ctrsm_kernel_LCTS,
   ctrsm_kernel_RNTS, ctrsm_kernel_RTTS, ctrsm_kernel_RRTS, ctrsm_kernel_RCTS,
 
@@ -533,6 +563,18 @@ gotoblas_t TABLE_NAME = {
 #endif
   zgemm_oncopyTS, zgemm_otcopyTS,
 
+#ifdef SMALL_MATRIX_OPT
+  zgemm_small_matrix_permitTS,
+  zgemm_small_kernel_nnTS, zgemm_small_kernel_ntTS, zgemm_small_kernel_nrTS, zgemm_small_kernel_ncTS,
+  zgemm_small_kernel_tnTS, zgemm_small_kernel_ttTS, zgemm_small_kernel_trTS, zgemm_small_kernel_tcTS,
+  zgemm_small_kernel_rnTS, zgemm_small_kernel_rtTS, zgemm_small_kernel_rrTS, zgemm_small_kernel_rcTS,
+  zgemm_small_kernel_cnTS, zgemm_small_kernel_ctTS, zgemm_small_kernel_crTS, zgemm_small_kernel_ccTS,
+  zgemm_small_kernel_b0_nnTS, zgemm_small_kernel_b0_ntTS, zgemm_small_kernel_b0_nrTS, zgemm_small_kernel_b0_ncTS,
+  zgemm_small_kernel_b0_tnTS, zgemm_small_kernel_b0_ttTS, zgemm_small_kernel_b0_trTS, zgemm_small_kernel_b0_tcTS,
+  zgemm_small_kernel_b0_rnTS, zgemm_small_kernel_b0_rtTS, zgemm_small_kernel_b0_rrTS, zgemm_small_kernel_b0_rcTS,
+  zgemm_small_kernel_b0_cnTS, zgemm_small_kernel_b0_ctTS, zgemm_small_kernel_b0_crTS, zgemm_small_kernel_b0_ccTS,
+#endif
+
   ztrsm_kernel_LNTS, ztrsm_kernel_LTTS, ztrsm_kernel_LRTS, ztrsm_kernel_LCTS,
   ztrsm_kernel_RNTS, ztrsm_kernel_RTTS, ztrsm_kernel_RRTS, ztrsm_kernel_RCTS,
 

kernel/x86_64/KERNEL.COOPERLAKE

@@ -1 +1,22 @@
 include $(KERNELDIR)/KERNEL.SKYLAKEX
+
+SBGEMM_SMALL_M_PERMIT = sbgemm_small_kernel_permit_cooperlake.c
+SBGEMM_SMALL_K_NN = sbgemm_small_kernel_nn_cooperlake.c
+SBGEMM_SMALL_K_B0_NN = sbgemm_small_kernel_nn_cooperlake.c
+SBGEMM_SMALL_K_NT = sbgemm_small_kernel_nt_cooperlake.c
+SBGEMM_SMALL_K_B0_NT = sbgemm_small_kernel_nt_cooperlake.c
+SBGEMM_SMALL_K_TN = sbgemm_small_kernel_tn_cooperlake.c
+SBGEMM_SMALL_K_B0_TN = sbgemm_small_kernel_tn_cooperlake.c
+SBGEMM_SMALL_K_TT = sbgemm_small_kernel_tt_cooperlake.c
+SBGEMM_SMALL_K_B0_TT = sbgemm_small_kernel_tt_cooperlake.c
+
+SBGEMM_BETA     = sgemm_beta_skylakex.c
+SBGEMMKERNEL    = sbgemm_kernel_16x4_cooperlake.c
+SBGEMMINCOPY    = sbgemm_ncopy_16_cooperlake.c
+SBGEMMITCOPY    = sbgemm_tcopy_16_cooperlake.c
+SBGEMMONCOPY    = sbgemm_ncopy_4_cooperlake.c
+SBGEMMOTCOPY    = sbgemm_tcopy_4_cooperlake.c
+SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
+SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
+SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
+SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)

kernel/x86_64/KERNEL.SKYLAKEX

@@ -10,6 +10,15 @@ STRSMKERNEL_LN =  ../generic/trsm_kernel_LN.c
 STRSMKERNEL_LT =  ../generic/trsm_kernel_LT.c
 STRSMKERNEL_RN =  ../generic/trsm_kernel_RN.c
 STRSMKERNEL_RT =  ../generic/trsm_kernel_RT.c
+SGEMM_SMALL_M_PERMIT = sgemm_small_kernel_permit_skylakex.c
+SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_skylakex.c
+SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_skylakex.c
+SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_skylakex.c
+SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_skylakex.c
+SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_skylakex.c
+SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_skylakex.c
+SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_skylakex.c
+SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_skylakex.c
 
 DGEMMKERNEL    =  dgemm_kernel_16x2_skylakex.c
 DTRMMKERNEL    =  dgemm_kernel_16x2_skylakex.c
@@ -18,6 +27,15 @@ DGEMMITCOPY    =  dgemm_tcopy_16_skylakex.c
 DGEMMONCOPY    =  ../generic/gemm_ncopy_2.c
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
 DTRSMKERNEL_RN =  ../generic/trsm_kernel_RN.c
+DGEMM_SMALL_M_PERMIT = dgemm_small_kernel_permit_skylakex.c
+DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_skylakex.c
+DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_skylakex.c
+DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_skylakex.c
+DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_skylakex.c
+DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_skylakex.c
+DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_skylakex.c
+DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_skylakex.c
+DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_skylakex.c
 
 SGEMM_BETA = sgemm_beta_skylakex.c
 DGEMM_BETA = dgemm_beta_skylakex.c

kernel/x86_64/bf16_common_macros.h

@@ -29,6 +29,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <immintrin.h>
 
+#define _MM512_BROADCASTD_EPI32(addr, zmm)             \
+                    __asm__ ("vpbroadcastd (%1), %0;"  \
+                            : "=v" (zmm)               \
+                            : "r"  (addr) )
+
+#define PREFETCH_T0(addr)             \
+                    __asm__ ("prefetcht0 (%0);"  \
+                            :                    \
+                            : "r"  (addr) )
+
 #define EXTRACT_LOW_256_FROM_512_2X(reg256, reg512)   \
     reg256##_0 = _mm512_castps512_ps256(reg512##_0);  \
     reg256##_1 = _mm512_castps512_ps256(reg512##_1);
@@ -46,25 +56,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #define BF16_MATRIX_LOAD_8x16(regArray, a, lda, idx_m, idx_n)      \
-    regArray##_0 = _mm256_loadu_si256(&a[(idx_m+0)*lda + idx_n]);  \
-    regArray##_1 = _mm256_loadu_si256(&a[(idx_m+1)*lda + idx_n]);  \
-    regArray##_2 = _mm256_loadu_si256(&a[(idx_m+2)*lda + idx_n]);  \
-    regArray##_3 = _mm256_loadu_si256(&a[(idx_m+3)*lda + idx_n]);  \
-    regArray##_4 = _mm256_loadu_si256(&a[(idx_m+4)*lda + idx_n]);  \
-    regArray##_5 = _mm256_loadu_si256(&a[(idx_m+5)*lda + idx_n]);  \
-    regArray##_6 = _mm256_loadu_si256(&a[(idx_m+6)*lda + idx_n]);  \
-    regArray##_7 = _mm256_loadu_si256(&a[(idx_m+7)*lda + idx_n]);
+    regArray##_0 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+0)*lda + idx_n]));  \
+    regArray##_1 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+1)*lda + idx_n]));  \
+    regArray##_2 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+2)*lda + idx_n]));  \
+    regArray##_3 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+3)*lda + idx_n]));  \
+    regArray##_4 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+4)*lda + idx_n]));  \
+    regArray##_5 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+5)*lda + idx_n]));  \
+    regArray##_6 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+6)*lda + idx_n]));  \
+    regArray##_7 = _mm256_loadu_si256((__m256i *)(&a[(idx_m+7)*lda + idx_n]));
 
 
 #define BF16_MATRIX_LOAD_8x8(regArray, a, lda, idx_m, idx_n)    \
-    regArray##_0 = _mm_loadu_si128(&a[(idx_m+0)*lda + idx_n]);  \
-    regArray##_1 = _mm_loadu_si128(&a[(idx_m+1)*lda + idx_n]);  \
-    regArray##_2 = _mm_loadu_si128(&a[(idx_m+2)*lda + idx_n]);  \
-    regArray##_3 = _mm_loadu_si128(&a[(idx_m+3)*lda + idx_n]);  \
-    regArray##_4 = _mm_loadu_si128(&a[(idx_m+4)*lda + idx_n]);  \
-    regArray##_5 = _mm_loadu_si128(&a[(idx_m+5)*lda + idx_n]);  \
-    regArray##_6 = _mm_loadu_si128(&a[(idx_m+6)*lda + idx_n]);  \
-    regArray##_7 = _mm_loadu_si128(&a[(idx_m+7)*lda + idx_n]);
+    regArray##_0 = _mm_loadu_si128((__m128i *)(&a[(idx_m+0)*lda + idx_n]));  \
+    regArray##_1 = _mm_loadu_si128((__m128i *)(&a[(idx_m+1)*lda + idx_n]));  \
+    regArray##_2 = _mm_loadu_si128((__m128i *)(&a[(idx_m+2)*lda + idx_n]));  \
+    regArray##_3 = _mm_loadu_si128((__m128i *)(&a[(idx_m+3)*lda + idx_n]));  \
+    regArray##_4 = _mm_loadu_si128((__m128i *)(&a[(idx_m+4)*lda + idx_n]));  \
+    regArray##_5 = _mm_loadu_si128((__m128i *)(&a[(idx_m+5)*lda + idx_n]));  \
+    regArray##_6 = _mm_loadu_si128((__m128i *)(&a[(idx_m+6)*lda + idx_n]));  \
+    regArray##_7 = _mm_loadu_si128((__m128i *)(&a[(idx_m+7)*lda + idx_n]));
 
 
 #define BF16_MATRIX_LOAD_1x32(regArray, a, lda, idx_m, idx_n)       \
@@ -143,11 +153,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #define BF16_VECTOR_LOAD_1x16(reg, x, idx_n)     \
-    reg = _mm256_loadu_si256(x + idx_n);
+    reg = _mm256_loadu_si256((__m256i *)(x + idx_n));
 
 
 #define BF16_VECTOR_LOAD_1x8(reg, x, idx_n)      \
-    reg = _mm_loadu_si128(x + idx_n);
+    reg = _mm_loadu_si128((__m128i *)(x + idx_n));
 
 
 #define BF16_VECTOR_MASKZ_LOAD_1x32(reg, x, idx_n, mask)     \
@@ -721,6 +731,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     _mm_mask_storeu_ps(targetAddr, mask, regResult);
 
 
+/* Store 16 (result + y) to y
+*/
+#define STORE16_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr)          \
+    regResult = _mm512_add_ps(regResult, _mm512_loadu_ps(targetAddr));  \
+    _mm512_storeu_ps(targetAddr, regResult);
+
+
+/* Masked store 16 (result + y) to y
+*/
+#define STORE16_MASK_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr, mask)           \
+    regResult = _mm512_add_ps(regResult, _mm512_maskz_loadu_ps(mask, targetAddr));  \
+    _mm512_mask_storeu_ps(targetAddr, mask, regResult);
+
+
+/* Store 8 (result + y) to y
+*/
+#define STORE8_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr)           \
+    regResult = _mm256_add_ps(regResult, _mm256_loadu_ps(targetAddr));  \
+    _mm256_storeu_ps(targetAddr, regResult);
+
+
+/* Masked store 8 (result + y) to y
+*/
+#define STORE8_MASK_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr, mask)            \
+    regResult = _mm256_add_ps(regResult, _mm256_maskz_loadu_ps(mask, targetAddr));  \
+    _mm256_mask_storeu_ps(targetAddr, mask, regResult);
+
+
+/* Store 4 (result + y) to y
+*/
+#define STORE4_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr)     \
+    regResult = _mm_add_ps(regResult, _mm_loadu_ps(targetAddr));  \
+    _mm_storeu_ps(targetAddr, regResult);
+
+
+/* Masked store 4 (result + y) to y
+*/
+#define STORE4_MASK_COMPLETE_RESULT_ONE_ONE(regResult, targetAddr, mask)      \
+    regResult = _mm_add_ps(regResult, _mm_maskz_loadu_ps(mask, targetAddr));  \
+    _mm_mask_storeu_ps(targetAddr, mask, regResult);
+
+
 /* Store 16 (alpha * result) to y
 */
 #define STORE16_COMPLETE_RESULT_ALPHA(regResult, targetAddr)  \

kernel/x86_64/casum_microk_skylakex-2.c

@@ -15,7 +15,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
     
     if (n2 < 64) {
         __m128 accum_10, accum_11, accum_12, accum_13;
-        __m128 abs_mask1;
+        __m128 abs_mask1 = abs_mask1;
 
         accum_10 = _mm_setzero_ps();
         accum_11 = _mm_setzero_ps();

kernel/x86_64/dasum_microk_haswell-2.c

@@ -38,10 +38,10 @@ static FLOAT dasum_kernel(BLASLONG n, FLOAT *x1)
 
          __m256i abs_mask = _mm256_set1_epi64x(0x7fffffffffffffff);
         for (i = 0; i < tail_index_AVX2; i += 16) {
-            accum_0 += (__m256d)_mm256_and_si256(_mm256_load_si256(&x1[i+ 0]), abs_mask);
-            accum_1 += (__m256d)_mm256_and_si256(_mm256_load_si256(&x1[i+ 4]), abs_mask);
-            accum_2 += (__m256d)_mm256_and_si256(_mm256_load_si256(&x1[i+ 8]), abs_mask);
-            accum_3 += (__m256d)_mm256_and_si256(_mm256_load_si256(&x1[i+12]), abs_mask);
+            accum_0 += (__m256d)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+ 0]), abs_mask);
+            accum_1 += (__m256d)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+ 4]), abs_mask);
+            accum_2 += (__m256d)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+ 8]), abs_mask);
+            accum_3 += (__m256d)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+12]), abs_mask);
         }
 
         accum_0 = accum_0 + accum_1 + accum_2 + accum_3;
@@ -63,10 +63,10 @@ static FLOAT dasum_kernel(BLASLONG n, FLOAT *x1)
 
         __m128i abs_mask2 = _mm_set1_epi64x(0x7fffffffffffffff);
         for (i = tail_index_AVX2; i < tail_index_SSE; i += 8) {
-            accum_20 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 0]), abs_mask2);
-            accum_21 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 2]), abs_mask2);
-            accum_22 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 4]), abs_mask2);
-            accum_23 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 6]), abs_mask2);
+            accum_20 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 0]), abs_mask2);
+            accum_21 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 2]), abs_mask2);
+            accum_22 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 4]), abs_mask2);
+            accum_23 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 6]), abs_mask2);
         }
 
         accum_20 = accum_20 + accum_21 + accum_22 + accum_23;

kernel/x86_64/dasum_microk_skylakex-2.c

@@ -58,10 +58,10 @@ static FLOAT dasum_kernel(BLASLONG n, FLOAT *x1)
 
         __m128i abs_mask2 = _mm_set1_epi64x(0x7fffffffffffffff);
         for (i = tail_index_AVX512; i < tail_index_SSE; i += 8) {
-            accum_20 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 0]), abs_mask2);
-            accum_21 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 2]), abs_mask2);
-            accum_22 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 4]), abs_mask2);
-            accum_23 += (__m128d)_mm_and_si128(_mm_loadu_si128(&x1[i + 6]), abs_mask2);
+            accum_20 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 0]), abs_mask2);
+            accum_21 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 2]), abs_mask2);
+            accum_22 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 4]), abs_mask2);
+            accum_23 += (__m128d)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 6]), abs_mask2);
         }
 
         accum_20 = accum_20 + accum_21 + accum_22 + accum_23;

kernel/x86_64/dgemm_small_kernel_nn_skylakex.c

@@ -0,0 +1,590 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <immintrin.h>
+#include "common.h"
+#include <stdio.h>
+#include <memory.h>
+
+#define DECLARE_RESULT_512(M, N) __m512d result##M##N = _mm512_setzero_pd()
+#define LOAD_A_512(M, N) __m512d Aval##M = _mm512_loadu_pd(&A[lda * k + i + (M*8)])
+#define MASK_LOAD_A_512(M, N) __m512d Aval##M = _mm512_maskz_loadu_pd(mask, &A[lda * k + i + (M*8)])
+#define BROADCAST_LOAD_B_512(M, N) __m512d Bval##N = _mm512_broadcastsd_pd(_mm_load_pd1(&B[k + ldb * (j+N)]))
+#define MATMUL_512(M, N) result##M##N = _mm512_fmadd_pd(Aval##M, Bval##N, result##M##N)
+#if defined(B0)
+#define STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+			_mm512_storeu_pd(&C[(j+N)*ldc + i + (M*8)], result##M##N)
+#define MASK_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+			_mm512_mask_storeu_pd(&C[(j+N)*ldc + i + (M*8)], mask, result##M##N)
+#else
+#define STORE_512(M, N) \
+	result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(result##M##N):"r"(&C[(j+N)*ldc + i + (M*8)]), "v"(beta_512)); \
+	_mm512_storeu_pd(&C[(j+N)*ldc + i + (M*8)], result##M##N)
+#define MASK_STORE_512(M, N) \
+	result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+	asm("vfmadd231pd (%1), %2, %0 %{%3%}": "+v"(result##M##N):"r"(&C[(j+N)*ldc + i + (M*8)]), "v"(beta_512), "k"(mask)); \
+	_mm512_mask_storeu_pd(&C[(j+N)*ldc + i + (M*8)], mask, result##M##N)
+#endif
+
+#define LOAD_KA_512(M, N) __m512d Aval##M = _mm512_loadu_pd(&mbuf[(mi + M)*K + k]);
+#define LOAD_KB_512(M, N) __m512d Bval##N = _mm512_loadu_pd(&B[(j + N)*ldb + k])
+#define MASK_LOAD_KA_512(M, N) __m512d Aval##M = _mm512_maskz_loadu_pd(mask, &mbuf[(mi + M)*K + k])
+#define MASK_LOAD_KB_512(M, N) __m512d Bval##N = _mm512_maskz_loadu_pd(mask, &B[(j + N)*ldb + k])
+#define REDUCE_4(rr0, rr1, rr2, rr3) \
+	__m512d r0, r1, r2, r3, t0, t1, t2, t3;\
+	r0 = _mm512_unpacklo_pd(rr0, rr1); r1 = _mm512_unpackhi_pd(rr0, rr1); \
+	r2 = _mm512_unpacklo_pd(rr2, rr3); r3 = _mm512_unpackhi_pd(rr2, rr3); \
+	t0 = _mm512_permutex2var_pd(r0, idx_lo, r2); t1 = _mm512_permutex2var_pd(r1, idx_lo, r3); \
+	t2 = _mm512_permutex2var_pd(r0, idx_hi, r2); t3 = _mm512_permutex2var_pd(r1, idx_hi, r3); \
+	r0 = _mm512_add_pd(t0, t1); r1 = _mm512_add_pd(t2, t3); t0 = _mm512_add_pd(r0, r1); \
+	__m256d s0, s1; \
+	s0 = _mm512_extractf64x4_pd(t0, 0); s1 = _mm512_extractf64x4_pd(t0, 1); \
+	s0 = _mm256_add_pd(s0, s1); s0 = _mm256_mul_pd(alpha_256, s0);
+#define REDUCE_M4(N) REDUCE_4(result0##N, result1##N, result2##N, result3##N)
+#define REDUCE_N4(M) REDUCE_4(result##M##0, result##M##1, result##M##2, result##M##3)
+#if defined(B0)
+#define STORE_REDUCE(M, N) C[(j+N)*ldc + i + M] = alpha * _mm512_reduce_add_pd(result##M##N);
+#define STORE_REDUCE_M4(N) {\
+	REDUCE_M4(N) \
+	_mm256_storeu_pd(&C[(j + N)*ldc + i], s0); \
+}
+#define STORE_REDUCE_N4(M) {\
+	REDUCE_N4(M) \
+	_mm256_i64scatter_pd(&C[j*ldc + i + M], vindex_n, s0, 8); \
+}
+#else
+#define STORE_REDUCE(M, N) C[(j+N)*ldc + i + M] = alpha * _mm512_reduce_add_pd(result##M##N) + beta * C[(j+N)*ldc + i + M];
+#define STORE_REDUCE_M4(N) {\
+	REDUCE_M4(N) \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(s0):"r"(&C[(j + N)*ldc + i]), "v"(beta_256)); \
+	_mm256_storeu_pd(&C[(j + N)*ldc + i], s0); \
+}
+#define STORE_REDUCE_N4(M) {\
+	REDUCE_N4(M) \
+	s1 = _mm256_i64gather_pd(&C[j*ldc + i + M], vindex_n, 8); \
+	s0 = _mm256_fmadd_pd(s1, beta_256, s0); \
+	_mm256_i64scatter_pd(&C[j*ldc + i + M], vindex_n, s0, 8); \
+}
+#endif
+
+#if defined(B0)
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	// column major
+	BLASLONG i, j, k;
+
+	BLASLONG m32 = M & ~31;
+	BLASLONG m16 = M & ~15;
+	BLASLONG m8 = M & ~7;
+	BLASLONG m4 = M & ~3;
+	BLASLONG m2 = M & ~1;
+
+	BLASLONG n6 = N - (N % 6);
+	BLASLONG n4 = N & ~3;
+	BLASLONG n2 = N & ~1;
+
+
+	__m512d alpha_512 = _mm512_broadcastsd_pd(_mm_load_pd1(&alpha));
+#if !defined(B0)
+	__m512d beta_512 = _mm512_broadcastsd_pd(_mm_load_pd1(&beta));
+#endif
+
+	for (i = 0; i < m32; i += 32) {
+		for (j = 0; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+			STORE_512(0, 1); STORE_512(1, 1); STORE_512(2, 1); STORE_512(3, 1);
+			STORE_512(0, 2); STORE_512(1, 2); STORE_512(2, 2); STORE_512(3, 2);
+			STORE_512(0, 3); STORE_512(1, 3); STORE_512(2, 3); STORE_512(3, 3);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+			STORE_512(0, 1); STORE_512(1, 1); STORE_512(2, 1); STORE_512(3, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+		}
+	}
+	for (; i < m16; i += 16) {
+		for (j = 0; j < n6; j += 6) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+			DECLARE_RESULT_512(0, 4); DECLARE_RESULT_512(1, 4);
+			DECLARE_RESULT_512(0, 5); DECLARE_RESULT_512(1, 5);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3);
+				MATMUL_512(0, 4); MATMUL_512(1, 4);
+				MATMUL_512(0, 5); MATMUL_512(1, 5);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+			STORE_512(0, 1); STORE_512(1, 1);
+			STORE_512(0, 2); STORE_512(1, 2);
+			STORE_512(0, 3); STORE_512(1, 3);
+			STORE_512(0, 4); STORE_512(1, 4);
+			STORE_512(0, 5); STORE_512(1, 5);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+			STORE_512(0, 1); STORE_512(1, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+		}
+	}
+	for (; i < m8; i += 8) {
+		for (j = 0; j < n6; j += 6) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			DECLARE_RESULT_512(0, 4);
+			DECLARE_RESULT_512(0, 5);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+				MATMUL_512(0, 4);
+				MATMUL_512(0, 5);
+			}
+			STORE_512(0, 0);
+			STORE_512(0, 1);
+			STORE_512(0, 2);
+			STORE_512(0, 3);
+			STORE_512(0, 4);
+			STORE_512(0, 5);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			STORE_512(0, 0);
+			STORE_512(0, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0);
+			}
+			STORE_512(0, 0);
+		}
+	}
+	int mm = M - i;
+	if (!mm) return 0;
+	if (mm > 4 || K < 16) {
+		register __mmask8 mask asm("k1") = (1UL << mm) - 1;
+		for (j = 0; j < n6; j += 6) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			DECLARE_RESULT_512(0, 4);
+			DECLARE_RESULT_512(0, 5);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+				MATMUL_512(0, 4);
+				MATMUL_512(0, 5);
+			}
+			MASK_STORE_512(0, 0);
+			MASK_STORE_512(0, 1);
+			MASK_STORE_512(0, 2);
+			MASK_STORE_512(0, 3);
+			MASK_STORE_512(0, 4);
+			MASK_STORE_512(0, 5);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			MASK_STORE_512(0, 0);
+			MASK_STORE_512(0, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0);
+			}
+			MASK_STORE_512(0, 0);
+		}
+	} else {
+		/* M => [1, 4]
+		 *
+		 * This kernel use dot-like style to calc a value - C(x, y):
+		 * C(x, y) = A(x, 0)*B(0, y) + A(x, 1)*B(1, y) +....+ A(x, K)*B(K, y)
+		 *
+		 * Alloc a buf to copy rest of A as row major,
+		 * so memory access from 0 to K is continuous for both A & B.
+		 *
+		 * Loading to zmm and FMA 8 of k at one loop,
+		 * finally reduce_add zmm to a single float result in C(x, y).
+		 *
+		 * Note: performance is bad when K is small.
+		 */
+		FLOAT *mbuf = (FLOAT *) malloc(sizeof(FLOAT)*mm*K);
+		__mmask8 mask = (1UL << mm) - 1;
+		BLASLONG k8 = K & ~7;
+		BLASLONG k4 = K & ~3;
+		for (k = 0; k < k4; k += 4) {
+			__m256d  r0, r1, r2, r3;
+			__m256d  t0, t1, t2, t3;
+			r0 = _mm256_maskz_loadu_pd(mask, &A[i + lda*(0 + k)]);
+			r1 = _mm256_maskz_loadu_pd(mask, &A[i + lda*(1 + k)]);
+			r2 = _mm256_maskz_loadu_pd(mask, &A[i + lda*(2 + k)]);
+			r3 = _mm256_maskz_loadu_pd(mask, &A[i + lda*(3 + k)]);
+
+			t0 = _mm256_unpacklo_pd(r0, r1);
+			t1 = _mm256_unpackhi_pd(r0, r1);
+			t2 = _mm256_unpacklo_pd(r2, r3);
+			t3 = _mm256_unpackhi_pd(r2, r3);
+
+			r0 = _mm256_permute2f128_pd(t0, t2, 0x20);
+			r1 = _mm256_permute2f128_pd(t1, t3, 0x20);
+			r2 = _mm256_permute2f128_pd(t0, t2, 0x31);
+			r3 = _mm256_permute2f128_pd(t1, t3, 0x31);
+
+			switch (mm) {
+				case 4: _mm256_storeu_pd(&mbuf[k + 3*K], r3);
+				case 3: _mm256_storeu_pd(&mbuf[k + 2*K], r2);
+				case 2: _mm256_storeu_pd(&mbuf[k + 1*K], r1);
+				case 1: _mm256_storeu_pd(&mbuf[k + 0*K], r0);
+			}
+		}
+		for (; k < K; k++) {
+			for (int ii = 0; ii < mm; ii++) {
+				mbuf[k + ii*K] = A[i + lda*k + ii];
+			}
+		}
+		int mi = 0;
+		__m256d alpha_256 = _mm256_broadcast_sd(&alpha);
+#if !defined(B0)
+		__m256d beta_256 = _mm256_broadcast_sd(&beta);
+#endif
+		__m256i vindex_n = _mm256_set_epi64x(ldc*3, ldc*2, ldc*1, 0);
+		long long permute_table[] = {
+			0, 1, 0|8, 1|8, 4, 5, 4|8, 5|8,
+			2, 3, 2|8, 3|8, 6, 7, 6|8, 7|8,
+		};
+		__m512i idx_lo = _mm512_loadu_si512(permute_table);
+		__m512i idx_hi = _mm512_loadu_si512(permute_table + 8);
+		for (; i < m4; i += 4, mi += 4) {
+			for (j = 0; j < n4; j += 4) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+				DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+				DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+				}
+				STORE_REDUCE_M4(0); STORE_REDUCE_M4(1); STORE_REDUCE_M4(2); STORE_REDUCE_M4(3);
+			}
+			for (; j < n2; j += 2) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				}
+				STORE_REDUCE_M4(0); STORE_REDUCE_M4(1);
+			}
+			for (; j < N; j += 1) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+					LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+					MASK_LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				}
+				STORE_REDUCE_M4(0);
+			}
+
+		}
+		for (; i < m2; i += 2, mi += 2) {
+			for (j = 0; j < n4; j += 4) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+				DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3);
+				}
+				STORE_REDUCE_N4(0); STORE_REDUCE_N4(1);
+			}
+			for (; j < n2; j += 2) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+				}
+				STORE_REDUCE(0, 0); STORE_REDUCE(1, 0);
+				STORE_REDUCE(0, 1); STORE_REDUCE(1, 1);
+
+			}
+			for (; j < N; j += 1) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+					LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+					MASK_LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+				}
+				STORE_REDUCE(0, 0); STORE_REDUCE(1, 0);
+			}
+		}
+		for (; i < M; i += 1, mi += 1) {
+			for (j = 0; j < n4; j += 4) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				DECLARE_RESULT_512(0, 2);
+				DECLARE_RESULT_512(0, 3);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+					MATMUL_512(0, 2);
+					MATMUL_512(0, 3);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+					MATMUL_512(0, 2);
+					MATMUL_512(0, 3);
+				}
+				STORE_REDUCE_N4(0);
+			}
+			for (; j < n2; j += 2) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x);
+					LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x);
+					MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+				}
+				STORE_REDUCE(0, 0);
+				STORE_REDUCE(0, 1);
+
+			}
+			for (; j < N; j += 1) {
+				DECLARE_RESULT_512(0, 0);
+				for (k = 0; k < k8; k += 8) {
+					LOAD_KA_512(0, x);
+					LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0);
+				}
+				int remains = K - k;
+				if (remains) {
+					mask = (1UL << remains) - 1;
+					MASK_LOAD_KA_512(0, x);
+					MASK_LOAD_KB_512(x, 0);
+
+					MATMUL_512(0, 0);
+				}
+				STORE_REDUCE(0, 0);
+			}
+		}
+		free(mbuf);
+	}
+	return 0;
+}

kernel/x86_64/dgemm_small_kernel_nt_skylakex.c

@@ -0,0 +1,535 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <immintrin.h>
+#include "common.h"
+#include <stdio.h>
+#include <memory.h>
+
+#define DECLARE_RESULT_512(M, N) __m512d result##M##N = _mm512_setzero_pd()
+#define LOAD_A_512(M, N) __m512d Aval##M = _mm512_loadu_pd(&A[lda * k + i + (M*8)])
+#define MASK_LOAD_A_512(M, N) __m512d Aval##M = _mm512_maskz_loadu_pd(mask, &A[lda * k + i + (M*8)])
+#define BROADCAST_LOAD_B_512(M, N) __m512d Bval##N = _mm512_broadcastsd_pd(_mm_load_sd(&B[ldb * k + j + N]))
+#define MATMUL_512(M, N) result##M##N = _mm512_fmadd_pd(Aval##M, Bval##N, result##M##N)
+
+#define BROADCAST_LOAD_A_512(M, N) __m512d Aval##M = _mm512_broadcastsd_pd(_mm_load_sd(&A[lda * k + i + M]))
+#define LOAD_B_512(M, N) __m512d Bval##N = _mm512_loadu_pd(&B[ldb * k + j + (N*8)])
+#define MASK_LOAD_B_512(M, N) __m512d Bval##N = _mm512_maskz_loadu_pd(mask, &B[ldb * k + j + (N*8)])
+#if defined(B0)
+#define STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+			_mm512_storeu_pd(&C[(j+N)*ldc + i + (M*8)], result##M##N)
+#define MASK_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+			_mm512_mask_storeu_pd(&C[(j+N)*ldc + i + (M*8)], mask, result##M##N)
+#define SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				_mm512_i64scatter_pd(&C[(j + N*8)*ldc + i + M], vindex_n, result##M##N, 8);
+#define MASK_SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				_mm512_mask_i64scatter_pd(&C[(j + N*8)*ldc + i + M], mask, vindex_n, result##M##N, 8)
+#else
+#define STORE_512(M, N) \
+	result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(result##M##N):"r"(&C[(j+N)*ldc + i + (M*8)]), "v"(beta_512)); \
+	_mm512_storeu_pd(&C[(j+N)*ldc + i + (M*8)], result##M##N)
+#define MASK_STORE_512(M, N) \
+	result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+	asm("vfmadd231pd (%1), %2, %0 %{%3%}": "+v"(result##M##N):"r"(&C[(j+N)*ldc + i + (M*8)]), "v"(beta_512), "k"(mask)); \
+	_mm512_mask_storeu_pd(&C[(j+N)*ldc + i + (M*8)], mask, result##M##N)
+#define SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				__m512d tmp##M##N = _mm512_i64gather_pd(vindex_n, &C[(j + N*8)*ldc + i + M], 8); \
+				result##M##N = _mm512_fmadd_pd(tmp##M##N, beta_512, result##M##N); \
+				_mm512_i64scatter_pd(&C[(j + N*8)*ldc + i + M], vindex_n, result##M##N, 8);
+#define MASK_SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				__m512d tmp##M##N = _mm512_mask_i64gather_pd(_mm512_setzero_pd(), mask, vindex_n, &C[(j + N*8)*ldc + i + M], 8); \
+				result##M##N = _mm512_fmadd_pd(tmp##M##N, beta_512, result##M##N); \
+				_mm512_mask_i64scatter_pd(&C[(j + N*8)*ldc + i + M], mask, vindex_n, result##M##N, 8);
+#endif
+
+#if defined(B0)
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	// column major
+	BLASLONG i, j, k;
+
+	BLASLONG m32 = M & ~31;
+	BLASLONG m16 = M & ~15;
+	BLASLONG m8 = M & ~7;
+	BLASLONG m4 = M & ~3;
+	BLASLONG m2 = M & ~1;
+
+	BLASLONG n32 = N & ~31;
+	BLASLONG n16 = N & ~15;
+	BLASLONG n8 = N & ~7;
+	BLASLONG n6 = N - (N % 6);
+	BLASLONG n4 = N & ~3;
+	BLASLONG n2 = N & ~1;
+
+
+	__m512d alpha_512 = _mm512_broadcastsd_pd(_mm_load_sd(&alpha));
+#if !defined(B0)
+	__m512d beta_512 = _mm512_broadcastsd_pd(_mm_load_sd(&beta));
+#endif
+
+	for (i = 0; i < m32; i += 32) {
+		for (j = 0; j < n6; j += 6) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+			DECLARE_RESULT_512(0, 4); DECLARE_RESULT_512(1, 4); DECLARE_RESULT_512(2, 4); DECLARE_RESULT_512(3, 4);
+			DECLARE_RESULT_512(0, 5); DECLARE_RESULT_512(1, 5); DECLARE_RESULT_512(2, 5); DECLARE_RESULT_512(3, 5);
+
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+				MATMUL_512(0, 4); MATMUL_512(1, 4); MATMUL_512(2, 4); MATMUL_512(3, 4);
+				MATMUL_512(0, 5); MATMUL_512(1, 5); MATMUL_512(2, 5); MATMUL_512(3, 5);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+			STORE_512(0, 1); STORE_512(1, 1); STORE_512(2, 1); STORE_512(3, 1);
+			STORE_512(0, 2); STORE_512(1, 2); STORE_512(2, 2); STORE_512(3, 2);
+			STORE_512(0, 3); STORE_512(1, 3); STORE_512(2, 3); STORE_512(3, 3);
+			STORE_512(0, 4); STORE_512(1, 4); STORE_512(2, 4); STORE_512(3, 4);
+			STORE_512(0, 5); STORE_512(1, 5); STORE_512(2, 5); STORE_512(3, 5);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+			STORE_512(0, 1); STORE_512(1, 1); STORE_512(2, 1); STORE_512(3, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x); LOAD_A_512(2, x); LOAD_A_512(3, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+			}
+			STORE_512(0, 0); STORE_512(1, 0); STORE_512(2, 0); STORE_512(3, 0);
+		}
+	}
+	for (; i < m16; i += 16) {
+		for (j = 0; j < n8; j += 8) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+			DECLARE_RESULT_512(0, 4); DECLARE_RESULT_512(1, 4);
+			DECLARE_RESULT_512(0, 5); DECLARE_RESULT_512(1, 5);
+			DECLARE_RESULT_512(0, 6); DECLARE_RESULT_512(1, 6);
+			DECLARE_RESULT_512(0, 7); DECLARE_RESULT_512(1, 7);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+				BROADCAST_LOAD_B_512(x, 6); BROADCAST_LOAD_B_512(x, 7);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3);
+				MATMUL_512(0, 4); MATMUL_512(1, 4);
+				MATMUL_512(0, 5); MATMUL_512(1, 5);
+				MATMUL_512(0, 6); MATMUL_512(1, 6);
+				MATMUL_512(0, 7); MATMUL_512(1, 7);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+			STORE_512(0, 1); STORE_512(1, 1);
+			STORE_512(0, 2); STORE_512(1, 2);
+			STORE_512(0, 3); STORE_512(1, 3);
+			STORE_512(0, 4); STORE_512(1, 4);
+			STORE_512(0, 5); STORE_512(1, 5);
+			STORE_512(0, 6); STORE_512(1, 6);
+			STORE_512(0, 7); STORE_512(1, 7);
+		}
+		for (;j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+			STORE_512(0, 1); STORE_512(1, 1);
+			STORE_512(0, 2); STORE_512(1, 2);
+			STORE_512(0, 3); STORE_512(1, 3);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+			STORE_512(0, 1); STORE_512(1, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x); LOAD_A_512(1, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+			}
+			STORE_512(0, 0); STORE_512(1, 0);
+		}
+	}
+	for (; i < m8; i += 8) {
+		for (j = 0; j < n8; j += 8) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			DECLARE_RESULT_512(0, 4);
+			DECLARE_RESULT_512(0, 5);
+			DECLARE_RESULT_512(0, 6);
+			DECLARE_RESULT_512(0, 7);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+				BROADCAST_LOAD_B_512(x, 6); BROADCAST_LOAD_B_512(x, 7);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+				MATMUL_512(0, 4);
+				MATMUL_512(0, 5);
+				MATMUL_512(0, 6);
+				MATMUL_512(0, 7);
+			}
+			STORE_512(0, 0);
+			STORE_512(0, 1);
+			STORE_512(0, 2);
+			STORE_512(0, 3);
+			STORE_512(0, 4);
+			STORE_512(0, 5);
+			STORE_512(0, 6);
+			STORE_512(0, 7);
+		}
+		for (; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+			}
+			STORE_512(0, 0);
+			STORE_512(0, 1);
+			STORE_512(0, 2);
+			STORE_512(0, 3);
+		}
+
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			STORE_512(0, 0);
+			STORE_512(0, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0);
+			for (k = 0; k < K; k++) {
+				LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0);
+			}
+			STORE_512(0, 0);
+		}
+	}
+	int mm = M - i;
+	if (mm >= 6) {
+		register __mmask16 mask asm("k1") = (1UL << mm) - 1;
+		for (j = 0; j < n8; j += 8) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			DECLARE_RESULT_512(0, 4);
+			DECLARE_RESULT_512(0, 5);
+			DECLARE_RESULT_512(0, 6);
+			DECLARE_RESULT_512(0, 7);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+				BROADCAST_LOAD_B_512(x, 4); BROADCAST_LOAD_B_512(x, 5);
+				BROADCAST_LOAD_B_512(x, 6); BROADCAST_LOAD_B_512(x, 7);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+				MATMUL_512(0, 4);
+				MATMUL_512(0, 5);
+				MATMUL_512(0, 6);
+				MATMUL_512(0, 7);
+			}
+			MASK_STORE_512(0, 0);
+			MASK_STORE_512(0, 1);
+			MASK_STORE_512(0, 2);
+			MASK_STORE_512(0, 3);
+			MASK_STORE_512(0, 4);
+			MASK_STORE_512(0, 5);
+			MASK_STORE_512(0, 6);
+			MASK_STORE_512(0, 7);
+		}
+		for (; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				BROADCAST_LOAD_B_512(x, 2); BROADCAST_LOAD_B_512(x, 3);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+			}
+			MASK_STORE_512(0, 0);
+			MASK_STORE_512(0, 1);
+			MASK_STORE_512(0, 2);
+			MASK_STORE_512(0, 3);
+		}
+
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0); BROADCAST_LOAD_B_512(x, 1);
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			MASK_STORE_512(0, 0);
+			MASK_STORE_512(0, 1);
+		}
+		for (; j < N; j++) {
+			DECLARE_RESULT_512(0, 0);
+			for (k = 0; k < K; k++) {
+				MASK_LOAD_A_512(0, x);
+				BROADCAST_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0);
+			}
+			MASK_STORE_512(0, 0);
+		}
+	} else if (mm > 0) {
+		long long index_n[8];
+		for (int ii = 0; ii < 8; ii++) {
+			index_n[ii] = ii * ldc;
+		}
+		__m512i vindex_n = _mm512_loadu_si512(index_n);
+		for (; i < m4; i += 4) {
+			for (j = 0; j < n32; j += 32) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+				DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+				DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					LOAD_B_512(x, 2);
+					LOAD_B_512(x, 3);
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0); SCATTER_STORE_512(2, 0); SCATTER_STORE_512(3, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1); SCATTER_STORE_512(2, 1); SCATTER_STORE_512(3, 1);
+				SCATTER_STORE_512(0, 2); SCATTER_STORE_512(1, 2); SCATTER_STORE_512(2, 2); SCATTER_STORE_512(3, 2);
+				SCATTER_STORE_512(0, 3); SCATTER_STORE_512(1, 3); SCATTER_STORE_512(2, 3); SCATTER_STORE_512(3, 3);
+			}
+			for (; j < n16; j += 16) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0); SCATTER_STORE_512(2, 0); SCATTER_STORE_512(3, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1); SCATTER_STORE_512(2, 1); SCATTER_STORE_512(3, 1);
+			}
+			__mmask8 mask = 0xff;
+			for (; j < N; j += 8) {
+				int remains = N - j;
+				if (remains < 8) mask = (1UL << remains) - 1;
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+					MASK_LOAD_B_512(x, 0);
+					MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				}
+				MASK_SCATTER_STORE_512(0, 0); MASK_SCATTER_STORE_512(1, 0); MASK_SCATTER_STORE_512(2, 0); MASK_SCATTER_STORE_512(3, 0);
+			}
+		}
+		for (; i < m2; i += 2) {
+			for (j = 0; j < n32; j += 32) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+				DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					LOAD_B_512(x, 2);
+					LOAD_B_512(x, 3);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1);
+				SCATTER_STORE_512(0, 2); SCATTER_STORE_512(1, 2);
+				SCATTER_STORE_512(0, 3); SCATTER_STORE_512(1, 3);
+			}
+			for (; j < n16; j += 16) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1);
+			}
+			__mmask8 mask = 0xff;
+			for (; j < N; j += 8) {
+				int remains = N - j;
+				if (remains < 8) mask = (1UL << remains) - 1;
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					MASK_LOAD_B_512(x, 0);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+				}
+				MASK_SCATTER_STORE_512(0, 0); MASK_SCATTER_STORE_512(1, 0);
+			}
+		}
+		for (; i < M; i += 1) {
+			for (j = 0; j < n32; j += 32) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				DECLARE_RESULT_512(0, 2);
+				DECLARE_RESULT_512(0, 3);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					LOAD_B_512(x, 2);
+					LOAD_B_512(x, 3);
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+					MATMUL_512(0, 2);
+					MATMUL_512(0, 3);
+				}
+				SCATTER_STORE_512(0, 0);
+				SCATTER_STORE_512(0, 1);
+				SCATTER_STORE_512(0, 2);
+				SCATTER_STORE_512(0, 3);
+			}
+			for (; j < n16; j += 16) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					LOAD_B_512(x, 0);
+					LOAD_B_512(x, 1);
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+				}
+				SCATTER_STORE_512(0, 0);
+				SCATTER_STORE_512(0, 1);
+			}
+			__mmask8 mask = 0xff;
+			for (; j < N; j += 8) {
+				int remains = N - j;
+				if (remains < 8) mask = (1UL << remains) - 1;
+				DECLARE_RESULT_512(0, 0);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					MASK_LOAD_B_512(x, 0);
+					MATMUL_512(0, 0);
+				}
+				MASK_SCATTER_STORE_512(0, 0);
+			}
+		}
+	}
+	return 0;
+}

kernel/x86_64/dgemm_small_kernel_permit_skylakex.c

@@ -0,0 +1,44 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta)
+{
+	double MNK = (double) M * (double) N * (double) K;
+	if (MNK > 100.0*100.0*100.0)  // disable for big size matrix
+		return 0;
+	if (transa && !transb) {
+		/* TN kernel perform not good when:
+		 * 1. C matrix is too big
+		 * 2. K is too small
+		 */
+		if (M * N > 1200 || K < 32)
+			return 0;
+	}
+	return 1;
+}

kernel/x86_64/dgemm_small_kernel_tn_skylakex.c

@@ -0,0 +1,322 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <immintrin.h>
+#include "common.h"
+#include <stdio.h>
+#include <memory.h>
+
+#define DECLARE_RESULT_512(M, N) __m512d result##M##N = _mm512_setzero_pd()
+#define MATMUL_512(M, N) result##M##N = _mm512_fmadd_pd(Aval##M, Bval##N, result##M##N)
+
+#define LOAD_KA_512(M, N) __m512d Aval##M = _mm512_loadu_pd(&A[(i + M)*lda + k]);
+#define LOAD_KB_512(M, N) __m512d Bval##N = _mm512_loadu_pd(&B[(j + N)*ldb + k])
+#define MASK_LOAD_KA_512(M, N) __m512d Aval##M = _mm512_maskz_loadu_pd(mask, &A[(i + M)*lda + k])
+#define MASK_LOAD_KB_512(M, N) __m512d Bval##N = _mm512_maskz_loadu_pd(mask, &B[(j + N)*ldb + k])
+
+#define REDUCE_4(rr0, rr1, rr2, rr3) \
+	__m512d r0, r1, r2, r3, t0, t1, t2, t3;\
+	r0 = _mm512_unpacklo_pd(rr0, rr1); r1 = _mm512_unpackhi_pd(rr0, rr1); \
+	r2 = _mm512_unpacklo_pd(rr2, rr3); r3 = _mm512_unpackhi_pd(rr2, rr3); \
+	t0 = _mm512_permutex2var_pd(r0, idx_lo, r2); t1 = _mm512_permutex2var_pd(r1, idx_lo, r3); \
+	t2 = _mm512_permutex2var_pd(r0, idx_hi, r2); t3 = _mm512_permutex2var_pd(r1, idx_hi, r3); \
+	r0 = _mm512_add_pd(t0, t1); r1 = _mm512_add_pd(t2, t3); t0 = _mm512_add_pd(r0, r1); \
+	__m256d s0, s1; \
+	s0 = _mm512_extractf64x4_pd(t0, 0); s1 = _mm512_extractf64x4_pd(t0, 1); \
+	s0 = _mm256_add_pd(s0, s1); s0 = _mm256_mul_pd(alpha_256, s0);
+
+#define REDUCE_M4(N) REDUCE_4(result0##N, result1##N, result2##N, result3##N)
+#define REDUCE_N4(M) REDUCE_4(result##M##0, result##M##1, result##M##2, result##M##3)
+
+#if defined(B0)
+#define STORE_REDUCE(M, N) C[(j+N)*ldc + i + M] = alpha * _mm512_reduce_add_pd(result##M##N)
+#define STORE_M4(N, s0) _mm256_storeu_pd(&C[(j + N)*ldc + i], s0);
+#define STORE_N4(M, s0) _mm256_i64scatter_pd(&C[j*ldc + i + M], vindex_n, s0, 8);
+#else
+#define STORE_REDUCE(M, N) C[(j+N)*ldc + i + M] = alpha * _mm512_reduce_add_pd(result##M##N) + beta * C[(j+N)*ldc + i + M]
+#define STORE_M4(N, s0) \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(s0):"r"(&C[(j + N)*ldc + i]), "v"(beta_256)); \
+	_mm256_storeu_pd(&C[(j + N)*ldc + i], s0);
+
+#define STORE_N4(M, s0) \
+	s0 = _mm256_fmadd_pd(_mm256_i64gather_pd(&C[j*ldc + i + M], vindex_n, 8), beta_256, s0); \
+	_mm256_i64scatter_pd(&C[j*ldc + i + M], vindex_n, s0, 8);
+#endif
+#define STORE_REDUCE_M4(N) {\
+	REDUCE_M4(N) \
+	STORE_M4(N, s0) \
+}
+#define STORE_REDUCE_N4(M) {\
+	REDUCE_N4(M) \
+	STORE_N4(M, s0) \
+}
+
+
+#if defined(B0)
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	// column major
+	BLASLONG i, j, k;
+
+	BLASLONG m4 = M & ~3;
+	BLASLONG m2 = M & ~1;
+
+	BLASLONG n4 = N & ~3;
+	BLASLONG n2 = N & ~1;
+
+	BLASLONG k8 = K & ~7;
+
+	__mmask8 mask;
+
+	__m256i vindex_n = _mm256_set_epi64x(ldc*3, ldc*2, ldc, 0);
+	__m256d alpha_256 = _mm256_broadcast_sd(&alpha);
+#if !defined(B0)
+	__m256d beta_256 = _mm256_broadcast_sd(&beta);
+#endif
+
+	long long permute_table[] = {
+		0, 1, 0|8, 1|8, 4, 5, 4|8, 5|8,
+		2, 3, 2|8, 3|8, 6, 7, 6|8, 7|8,
+	};
+	__m512i idx_lo = _mm512_loadu_si512(permute_table);
+	__m512i idx_hi = _mm512_loadu_si512(permute_table + 8);
+
+	for (i = 0; i < m4; i += 4) {
+		for (j = 0; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+			}
+			STORE_REDUCE_M4(0); STORE_REDUCE_M4(1); STORE_REDUCE_M4(2); STORE_REDUCE_M4(3);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+			}
+			STORE_REDUCE_M4(0); STORE_REDUCE_M4(1);
+		}
+		for (; j < N; j += 1) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x); LOAD_KA_512(2, x); LOAD_KA_512(3, x);
+				LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x); MASK_LOAD_KA_512(2, x); MASK_LOAD_KA_512(3, x);
+				MASK_LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+			}
+			STORE_REDUCE_M4(0);
+		}
+
+	}
+	for (; i < m2; i += 2) {
+		for (j = 0; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3);
+			}
+			STORE_REDUCE_N4(0); STORE_REDUCE_N4(1);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1);
+			}
+			STORE_REDUCE(0, 0); STORE_REDUCE(1, 0);
+			STORE_REDUCE(0, 1); STORE_REDUCE(1, 1);
+
+		}
+		for (; j < N; j += 1) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x); LOAD_KA_512(1, x);
+				LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x); MASK_LOAD_KA_512(1, x);
+				MASK_LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0); MATMUL_512(1, 0);
+			}
+			STORE_REDUCE(0, 0); STORE_REDUCE(1, 0);
+		}
+	}
+	for (; i < M; i += 1) {
+		for (j = 0; j < n4; j += 4) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			DECLARE_RESULT_512(0, 2);
+			DECLARE_RESULT_512(0, 3);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1); LOAD_KB_512(x, 2); LOAD_KB_512(x, 3);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1); MASK_LOAD_KB_512(x, 2); MASK_LOAD_KB_512(x, 3);
+
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+				MATMUL_512(0, 2);
+				MATMUL_512(0, 3);
+			}
+			STORE_REDUCE_N4(0);
+		}
+		for (; j < n2; j += 2) {
+			DECLARE_RESULT_512(0, 0);
+			DECLARE_RESULT_512(0, 1);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x);
+				LOAD_KB_512(x, 0); LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x);
+				MASK_LOAD_KB_512(x, 0); MASK_LOAD_KB_512(x, 1);
+
+				MATMUL_512(0, 0);
+				MATMUL_512(0, 1);
+			}
+			STORE_REDUCE(0, 0);
+			STORE_REDUCE(0, 1);
+
+		}
+		for (; j < N; j += 1) {
+			DECLARE_RESULT_512(0, 0);
+			for (k = 0; k < k8; k += 8) {
+				LOAD_KA_512(0, x);
+				LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0);
+			}
+			int remains = K - k;
+			if (remains) {
+				mask = (1UL << remains) - 1;
+				MASK_LOAD_KA_512(0, x);
+				MASK_LOAD_KB_512(x, 0);
+
+				MATMUL_512(0, 0);
+			}
+			STORE_REDUCE(0, 0);
+		}
+	}
+	return 0;
+}

kernel/x86_64/dgemm_small_kernel_tt_skylakex.c

@@ -0,0 +1,392 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <immintrin.h>
+#include "common.h"
+#include <stdio.h>
+
+#define DECLARE_RESULT_512(M, N) __m512d result##M##N = _mm512_setzero_pd()
+#define BROADCAST_LOAD_A_512(M, N) __m512d Aval##M = _mm512_broadcastsd_pd(_mm_load_sd(&A[k  + lda * (i+M)]))
+#define LOAD_B_512(M,N)  __m512d Bval##N = _mm512_loadu_pd(&B[ldb * k + j + (N*8)])
+#define MASK_LOAD_B_512(M, N) __m512d Bval##N = _mm512_maskz_loadu_pd(mask, &B[ldb * k + j + (N*8)])
+#define MATMUL_512(M, N) result##M##N = _mm512_fmadd_pd(Aval##M, Bval##N, result##M##N)
+
+#if defined(B0)
+#define STORE_8xy(v, N, x, y) _mm512_storeu_pd(&C[(j + N*8 + x + y*8)*ldc + i], v)
+#define STORE_4xy(v, N, x, y) _mm256_storeu_pd(&C[(j + N*8 + x + y*4)*ldc + i], v)
+#define SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				_mm512_i64scatter_pd(&C[(j + N*8)*ldc + i + M], vindex_n, result##M##N, 8);
+#define MASK_SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				    _mm512_mask_i64scatter_pd(&C[(j + N*8)*ldc + i + M], mask, vindex_n, result##M##N, 8);
+#else
+#define STORE_8xy(v, N, x, y) \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(v): "r"(&C[(j + N*8 + x + y*8)*ldc + i]), "v"(beta_512)); \
+	_mm512_storeu_pd(&C[(j + N*8 + x + y*8)*ldc + i], v)
+#define STORE_4xy(v, N, x, y) \
+	asm("vfmadd231pd (%1), %2, %0": "+v"(v): "r"(&C[(j + N*8 + x + y*4)*ldc + i]), "v"(beta_256)); \
+	_mm256_storeu_pd(&C[(j + N*8 + x + y*4)*ldc + i], v)
+#define SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				__m512d tmp##M##N = _mm512_i64gather_pd(vindex_n, &C[(j + N*8)*ldc + i + M], 8); \
+				result##M##N = _mm512_fmadd_pd(tmp##M##N, beta_512, result##M##N); \
+				_mm512_i64scatter_pd(&C[(j + N*8)*ldc + i + M], vindex_n, result##M##N, 8);
+#define MASK_SCATTER_STORE_512(M, N) result##M##N = _mm512_mul_pd(result##M##N, alpha_512); \
+				__m512d tmp##M##N = _mm512_mask_i64gather_pd(_mm512_setzero_pd(), mask, vindex_n, &C[(j + N*8)*ldc + i + M], 8); \
+				result##M##N = _mm512_fmadd_pd(tmp##M##N, beta_512, result##M##N); \
+				_mm512_mask_i64scatter_pd(&C[(j + N*8)*ldc + i + M], mask, vindex_n, result##M##N, 8);
+#endif
+
+#define REORDER_8x8(r0, r1, r2, r3, r4, r5, r6, r7) \
+	__m512d t0, t1, t2, t3, t4, t5, t6, t7; \
+	t0 = _mm512_unpacklo_pd(r0, r1); \
+	t1 = _mm512_unpackhi_pd(r0, r1); \
+	t2 = _mm512_unpacklo_pd(r2, r3); \
+	t3 = _mm512_unpackhi_pd(r2, r3); \
+	t4 = _mm512_unpacklo_pd(r4, r5); \
+	t5 = _mm512_unpackhi_pd(r4, r5); \
+	t6 = _mm512_unpacklo_pd(r6, r7); \
+	t7 = _mm512_unpackhi_pd(r6, r7); \
+	r0 = _mm512_shuffle_f64x2(t0, t2, 0x88); \
+	r1 = _mm512_shuffle_f64x2(t1, t3, 0x88); \
+	r2 = _mm512_shuffle_f64x2(t0, t2, 0xdd); \
+	r3 = _mm512_shuffle_f64x2(t1, t3, 0xdd); \
+	r4 = _mm512_shuffle_f64x2(t4, t6, 0x88); \
+	r5 = _mm512_shuffle_f64x2(t5, t7, 0x88); \
+	r6 = _mm512_shuffle_f64x2(t4, t6, 0xdd); \
+	r7 = _mm512_shuffle_f64x2(t5, t7, 0xdd); \
+	t0 = _mm512_permutex2var_pd(r0, idx_lo, r4); \
+	t1 = _mm512_permutex2var_pd(r1, idx_lo, r5); \
+	t2 = _mm512_permutex2var_pd(r2, idx_lo, r6); \
+	t3 = _mm512_permutex2var_pd(r3, idx_lo, r7); \
+	t4 = _mm512_permutex2var_pd(r0, idx_hi, r4); \
+	t5 = _mm512_permutex2var_pd(r1, idx_hi, r5); \
+	t6 = _mm512_permutex2var_pd(r2, idx_hi, r6); \
+	t7 = _mm512_permutex2var_pd(r3, idx_hi, r7); \
+	t0 = _mm512_mul_pd(t0, alpha_512); \
+	t1 = _mm512_mul_pd(t1, alpha_512); \
+	t2 = _mm512_mul_pd(t2, alpha_512); \
+	t3 = _mm512_mul_pd(t3, alpha_512); \
+	t4 = _mm512_mul_pd(t4, alpha_512); \
+	t5 = _mm512_mul_pd(t5, alpha_512); \
+	t6 = _mm512_mul_pd(t6, alpha_512); \
+	t7 = _mm512_mul_pd(t7, alpha_512);
+
+#define SAVE_8(N, x) {\
+	STORE_8xy(t##x, N, x, 0); \
+}
+
+#define REORDER_STORE_8x8(N) {\
+	REORDER_8x8(result0##N, result1##N, result2##N, result3##N, result4##N, result5##N, result6##N, result7##N); \
+	SAVE_8(N, 0); SAVE_8(N, 1); SAVE_8(N, 2); SAVE_8(N, 3); SAVE_8(N, 4); SAVE_8(N, 5); SAVE_8(N, 6); SAVE_8(N, 7); \
+}
+
+#define MASK_SAVE_8() \
+	switch (nn) { \
+		case 8: SAVE_8(0, 7); \
+		case 7: SAVE_8(0, 6); \
+		case 6: SAVE_8(0, 5); \
+		case 5: SAVE_8(0, 4); \
+		case 4: SAVE_8(0, 3); \
+		case 3: SAVE_8(0, 2); \
+		case 2: SAVE_8(0, 1); \
+		case 1: SAVE_8(0, 0); \
+	}
+
+#define MASK_REORDER_STORE_8x8(N) {\
+	REORDER_8x8(result0##N, result1##N, result2##N, result3##N, result4##N, result5##N, result6##N, result7##N); \
+	MASK_SAVE_8(); \
+}
+
+#define REORDER_4x8(r0, r1, r2, r3) \
+	__m512d t0, t1, t2, t3; \
+	t0 = _mm512_unpacklo_pd(r0, r1); \
+	t1 = _mm512_unpackhi_pd(r0, r1); \
+	t2 = _mm512_unpacklo_pd(r2, r3); \
+	t3 = _mm512_unpackhi_pd(r2, r3); \
+	r0 = _mm512_permutex2var_pd(t0, idx_lo, t2); \
+	r1 = _mm512_permutex2var_pd(t1, idx_lo, t3); \
+	r2 = _mm512_permutex2var_pd(t0, idx_hi, t2); \
+	r3 = _mm512_permutex2var_pd(t1, idx_hi, t3); \
+	t0 = _mm512_mul_pd(r0, alpha_512); \
+	t1 = _mm512_mul_pd(r1, alpha_512); \
+	t2 = _mm512_mul_pd(r2, alpha_512); \
+	t3 = _mm512_mul_pd(r3, alpha_512);
+
+#define SAVE_4(N, x, y) {\
+	__m256d v4 = _mm512_extractf64x4_pd(t##x, y); \
+	STORE_4xy(v4, N, x, y); \
+}
+
+#define REORDER_STORE_4x8(N) {\
+	REORDER_4x8(result0##N, result1##N, result2##N, result3##N); \
+	SAVE_4(N, 0, 0); SAVE_4(N, 1, 0); SAVE_4(N, 2, 0); SAVE_4(N, 3, 0); \
+	SAVE_4(N, 0, 1); SAVE_4(N, 1, 1); SAVE_4(N, 2, 1); SAVE_4(N, 3, 1); \
+}
+
+#define MASK_SAVE_4() \
+	switch (nn) { \
+		case 8: SAVE_4(0, 3, 1); \
+		case 7: SAVE_4(0, 2, 1); \
+		case 6: SAVE_4(0, 1, 1); \
+		case 5: SAVE_4(0, 0, 1); \
+		case 4: SAVE_4(0, 3, 0); \
+		case 3: SAVE_4(0, 2, 0); \
+		case 2: SAVE_4(0, 1, 0); \
+		case 1: SAVE_4(0, 0, 0); \
+	}
+
+#define MASK_REORDER_STORE_4x8(N) {\
+	REORDER_4x8(result0##N, result1##N, result2##N, result3##N); \
+	MASK_SAVE_4(); \
+}
+
+
+#if defined(B0)
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)
+#else
+int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT * A, BLASLONG lda, FLOAT alpha, FLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)
+#endif
+{
+	// column major
+	BLASLONG i, j, k;
+
+	BLASLONG m8 = M & ~7;
+	BLASLONG m4 = M & ~3;
+	BLASLONG m2 = M & ~1;
+
+	BLASLONG n32 = N & ~31;
+	BLASLONG n16 = N & ~15;
+
+	__m512d alpha_512 = _mm512_broadcastsd_pd(_mm_load_sd(&alpha));
+#if !defined(B0)
+	__m512d beta_512 = _mm512_broadcastsd_pd(_mm_load_sd(&beta));
+	__m256d beta_256 = _mm256_broadcastsd_pd(_mm_load_sd(&beta));
+#endif
+	long long permute_table[] = {
+		0, 1, 4, 5, 0|8, 1|8, 4|8, 5|8,
+		2, 3, 6, 7, 2|8, 3|8, 6|8, 7|8,
+	};
+	__m512i idx_lo = _mm512_loadu_si512(permute_table);
+	__m512i idx_hi = _mm512_loadu_si512(permute_table + 8);
+
+	for (i = 0; i < m8; i += 8) {
+		for (j = 0; j < n16; j += 16) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(4, 0); DECLARE_RESULT_512(5, 0); DECLARE_RESULT_512(6, 0); DECLARE_RESULT_512(7, 0);
+
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			DECLARE_RESULT_512(4, 1); DECLARE_RESULT_512(5, 1); DECLARE_RESULT_512(6, 1); DECLARE_RESULT_512(7, 1);
+			for (k = 0; k < K; k++) {
+				BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+				BROADCAST_LOAD_A_512(4, x); BROADCAST_LOAD_A_512(5, x); BROADCAST_LOAD_A_512(6, x); BROADCAST_LOAD_A_512(7, x);
+				LOAD_B_512(x, 0); LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(4, 0); MATMUL_512(5, 0); MATMUL_512(6, 0); MATMUL_512(7, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				MATMUL_512(4, 1); MATMUL_512(5, 1); MATMUL_512(6, 1); MATMUL_512(7, 1);
+			}
+			REORDER_STORE_8x8(0);
+			REORDER_STORE_8x8(1);
+		}
+		__mmask8 mask = 0xff;
+		int nn = 8;
+		for (; j < N; j += 8) {
+			if (N - j < 8) {
+				nn = N - j;
+				mask = (1UL << nn) - 1;
+			}
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(4, 0); DECLARE_RESULT_512(5, 0); DECLARE_RESULT_512(6, 0); DECLARE_RESULT_512(7, 0);
+			for (k = 0; k < K; k++) {
+				BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+				BROADCAST_LOAD_A_512(4, x); BROADCAST_LOAD_A_512(5, x); BROADCAST_LOAD_A_512(6, x); BROADCAST_LOAD_A_512(7, x);
+				MASK_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(4, 0); MATMUL_512(5, 0); MATMUL_512(6, 0); MATMUL_512(7, 0);
+			}
+			MASK_REORDER_STORE_8x8(0);
+		}
+	}
+	for (; i < m4; i += 4) {
+		long long permute_table2[] = {
+			0, 1, 0|8, 1|8, 4, 5, 4|8, 5|8,
+			2, 3, 2|8, 3|8, 6, 7, 6|8, 7|8,
+		};
+		idx_lo = _mm512_loadu_si512(permute_table2);
+		idx_hi = _mm512_loadu_si512(permute_table2 + 8);
+
+		for (j = 0; j < n32; j += 32) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2); DECLARE_RESULT_512(2, 2); DECLARE_RESULT_512(3, 2);
+			DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3); DECLARE_RESULT_512(2, 3); DECLARE_RESULT_512(3, 3);
+			for (k = 0; k < K; k++) {
+				BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+				LOAD_B_512(x, 0); LOAD_B_512(x, 1); LOAD_B_512(x, 2); LOAD_B_512(x, 3);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+				MATMUL_512(0, 2); MATMUL_512(1, 2); MATMUL_512(2, 2); MATMUL_512(3, 2);
+				MATMUL_512(0, 3); MATMUL_512(1, 3); MATMUL_512(2, 3); MATMUL_512(3, 3);
+			}
+			REORDER_STORE_4x8(0);
+			REORDER_STORE_4x8(1);
+			REORDER_STORE_4x8(2);
+			REORDER_STORE_4x8(3);
+		}
+		for (; j < n16; j += 16) {
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1); DECLARE_RESULT_512(2, 1); DECLARE_RESULT_512(3, 1);
+			for (k = 0; k < K; k++) {
+				BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+				LOAD_B_512(x, 0); LOAD_B_512(x, 1);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+				MATMUL_512(0, 1); MATMUL_512(1, 1); MATMUL_512(2, 1); MATMUL_512(3, 1);
+			}
+			REORDER_STORE_4x8(0);
+			REORDER_STORE_4x8(1);
+		}
+		__mmask8 mask = 0xff;
+		int nn = 8;
+		for (; j < N; j += 8) {
+			if (N - j < 8) {
+				nn = N - j;
+				mask = (1UL << nn) - 1;
+			}
+			DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0); DECLARE_RESULT_512(2, 0); DECLARE_RESULT_512(3, 0);
+			for (k = 0; k < K; k++) {
+				BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x); BROADCAST_LOAD_A_512(2, x); BROADCAST_LOAD_A_512(3, x);
+				MASK_LOAD_B_512(x, 0);
+				MATMUL_512(0, 0); MATMUL_512(1, 0); MATMUL_512(2, 0); MATMUL_512(3, 0);
+			}
+			MASK_REORDER_STORE_4x8(0);
+		}
+	}
+	if (i < M) {
+		long long index_n[8];
+		for (int ii = 0; ii < 8; ii++) {
+			index_n[ii] = ii * ldc;
+		}
+		__m512i vindex_n = _mm512_loadu_si512(index_n);
+#if !defined(B0)
+		__m512d beta_512 = _mm512_broadcastsd_pd(_mm_load_sd(&beta));
+#endif
+		for (; i < m2; i += 2) {
+			for (j = 0; j < n32; j += 32) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				DECLARE_RESULT_512(0, 2); DECLARE_RESULT_512(1, 2);
+				DECLARE_RESULT_512(0, 3); DECLARE_RESULT_512(1, 3);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					LOAD_B_512(x, 0); LOAD_B_512(x, 1); LOAD_B_512(x, 2); LOAD_B_512(x, 3);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+					MATMUL_512(0, 2); MATMUL_512(1, 2);
+					MATMUL_512(0, 3); MATMUL_512(1, 3);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1);
+				SCATTER_STORE_512(0, 2); SCATTER_STORE_512(1, 2);
+				SCATTER_STORE_512(0, 3); SCATTER_STORE_512(1, 3);
+			}
+			for (; j < n16; j += 16) {
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				DECLARE_RESULT_512(0, 1); DECLARE_RESULT_512(1, 1);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					LOAD_B_512(x, 0); LOAD_B_512(x, 1);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+					MATMUL_512(0, 1); MATMUL_512(1, 1);
+				}
+				SCATTER_STORE_512(0, 0); SCATTER_STORE_512(1, 0);
+				SCATTER_STORE_512(0, 1); SCATTER_STORE_512(1, 1);
+			}
+			__mmask8 mask = 0xff;
+			int nn = 8;
+			for (; j < N; j += 8) {
+				if (N - j < 8) {
+					nn = N - j;
+					mask = (1UL << nn) - 1;
+				}
+				DECLARE_RESULT_512(0, 0); DECLARE_RESULT_512(1, 0);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x); BROADCAST_LOAD_A_512(1, x);
+					MASK_LOAD_B_512(x, 0);
+					MATMUL_512(0, 0); MATMUL_512(1, 0);
+				}
+				MASK_SCATTER_STORE_512(0, 0); MASK_SCATTER_STORE_512(1, 0);
+			}
+		}
+		for (; i < M; i += 1) {
+			for (j = 0; j < n32; j += 32) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				DECLARE_RESULT_512(0, 2);
+				DECLARE_RESULT_512(0, 3);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					LOAD_B_512(x, 0); LOAD_B_512(x, 1); LOAD_B_512(x, 2); LOAD_B_512(x, 3);
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+					MATMUL_512(0, 2);
+					MATMUL_512(0, 3);
+				}
+				SCATTER_STORE_512(0, 0);
+				SCATTER_STORE_512(0, 1);
+				SCATTER_STORE_512(0, 2);
+				SCATTER_STORE_512(0, 3);
+			}
+			for (; j < n16; j += 16) {
+				DECLARE_RESULT_512(0, 0);
+				DECLARE_RESULT_512(0, 1);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					LOAD_B_512(x, 0); LOAD_B_512(x, 1);
+					MATMUL_512(0, 0);
+					MATMUL_512(0, 1);
+				}
+				SCATTER_STORE_512(0, 0);
+				SCATTER_STORE_512(0, 1);
+			}
+			__mmask8 mask = 0xff;
+			int nn = 8;
+			for (; j < N; j += 8) {
+				if (N - j < 8) {
+					nn = N - j;
+					mask = (1UL << nn) - 1;
+				}
+				DECLARE_RESULT_512(0, 0);
+				for (k = 0; k < K; k++) {
+					BROADCAST_LOAD_A_512(0, x);
+					MASK_LOAD_B_512(x, 0);
+					MATMUL_512(0, 0);
+				}
+				MASK_SCATTER_STORE_512(0, 0);
+			}
+		}
+	}
+	return 0;
+}

kernel/x86_64/sasum_microk_haswell-2.c

@@ -38,10 +38,10 @@ static FLOAT sasum_kernel(BLASLONG n, FLOAT *x1)
 
         __m256i abs_mask = _mm256_set1_epi32(0x7fffffff);
         for (i = 0; i < tail_index_AVX2; i += 32) {
-            accum_0 += (__m256)_mm256_and_si256(_mm256_load_si256(&x1[i+ 0]), abs_mask);
-            accum_1 += (__m256)_mm256_and_si256(_mm256_load_si256(&x1[i+ 8]), abs_mask);
-            accum_2 += (__m256)_mm256_and_si256(_mm256_load_si256(&x1[i+16]), abs_mask);
-            accum_3 += (__m256)_mm256_and_si256(_mm256_load_si256(&x1[i+24]), abs_mask);
+            accum_0 += (__m256)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+ 0]), abs_mask);
+            accum_1 += (__m256)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+ 8]), abs_mask);
+            accum_2 += (__m256)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+16]), abs_mask);
+            accum_3 += (__m256)_mm256_and_si256(_mm256_load_si256((__m256i*)&x1[i+24]), abs_mask);
         }
 
         accum_0 = accum_0 + accum_1 + accum_2 + accum_3;
@@ -62,8 +62,8 @@ static FLOAT sasum_kernel(BLASLONG n, FLOAT *x1)
 
         __m128i abs_mask2 = _mm_set1_epi32(0x7fffffff);
         for (i = tail_index_AVX2; i < tail_index_SSE; i += 8) {
-            accum_20 += (__m128)_mm_and_si128(_mm_loadu_si128(&x1[i + 0]), abs_mask2);
-            accum_21 += (__m128)_mm_and_si128(_mm_loadu_si128(&x1[i + 4]), abs_mask2);
+            accum_20 += (__m128)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 0]), abs_mask2);
+            accum_21 += (__m128)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 4]), abs_mask2);
         }
         
         accum_20 += accum_21;

kernel/x86_64/sasum_microk_skylakex-2.c

@@ -53,8 +53,8 @@ static FLOAT sasum_kernel(BLASLONG n, FLOAT *x1)
 
         __m128i abs_mask2 = _mm_set1_epi32(0x7fffffff);
         for (i = tail_index_AVX512; i < tail_index_SSE; i += 8) {
-            accum_20 += (__m128)_mm_and_si128(_mm_loadu_si128(&x1[i + 0]), abs_mask2);
-            accum_21 += (__m128)_mm_and_si128(_mm_loadu_si128(&x1[i + 4]), abs_mask2);
+            accum_20 += (__m128)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 0]), abs_mask2);
+            accum_21 += (__m128)_mm_and_si128(_mm_loadu_si128((__m128i*)&x1[i + 4]), abs_mask2);
         }
         
         accum_20 += accum_21;

kernel/x86_64/sbdot_microk_cooperlake.c

@@ -79,21 +79,21 @@ static float sbdot_accl_kernel(BLASLONG n, bfloat16 *x, bfloat16 *y)
         __m256 accum256_1 = _mm256_setzero_ps();
         int tail_index_32  = n&(~31);
         for (int j = 0; j < tail_index_32; j += 32) {
-            accum256   = _mm256_dpbf16_ps(accum256,   (__m256bh) _mm256_loadu_si256(&x[j+ 0]), (__m256bh) _mm256_loadu_si256(&y[j+ 0]));
-            accum256_1 = _mm256_dpbf16_ps(accum256_1, (__m256bh) _mm256_loadu_si256(&x[j+16]), (__m256bh) _mm256_loadu_si256(&y[j+16]));
+            accum256   = _mm256_dpbf16_ps(accum256,   (__m256bh) _mm256_loadu_si256((__m256i *)&x[j+ 0]), (__m256bh) _mm256_loadu_si256((__m256i *)&y[j+ 0]));
+            accum256_1 = _mm256_dpbf16_ps(accum256_1, (__m256bh) _mm256_loadu_si256((__m256i *)&x[j+16]), (__m256bh) _mm256_loadu_si256((__m256i *)&y[j+16]));
         }
         accum256 = _mm256_add_ps(accum256, accum256_1);
 
         /* Processing the remaining <32 chunk with 16-elements processing */
         if ((n&16) != 0) {
-            accum256 = _mm256_dpbf16_ps(accum256, (__m256bh) _mm256_loadu_si256(&x[tail_index_32]), (__m256bh) _mm256_loadu_si256(&y[tail_index_32]));
+            accum256 = _mm256_dpbf16_ps(accum256, (__m256bh) _mm256_loadu_si256((__m256i *)&x[tail_index_32]), (__m256bh) _mm256_loadu_si256((__m256i *)&y[tail_index_32]));
         }
         accum128 = _mm_add_ps(_mm256_castps256_ps128(accum256), _mm256_extractf128_ps(accum256, 1));
 
         /* Processing the remaining <16 chunk with 8-elements processing */
         if ((n&8) != 0) {
             int tail_index_16  = n&(~15);
-            accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128(&x[tail_index_16]), (__m128bh) _mm_loadu_si128(&y[tail_index_16]));
+            accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128((__m128i *)&x[tail_index_16]), (__m128bh) _mm_loadu_si128((__m128i *)&y[tail_index_16]));
         }
 
         /* Processing the remaining <8 chunk with masked 8-elements processing */
@@ -108,13 +108,13 @@ static float sbdot_accl_kernel(BLASLONG n, bfloat16 *x, bfloat16 *y)
     } else if (n > 15) { /* n range from 16 to 31 */
         /* Processing <32 chunk with 16-elements processing */
         __m256 accum256   = _mm256_setzero_ps();
-        accum256 = _mm256_dpbf16_ps(accum256, (__m256bh) _mm256_loadu_si256(&x[0]), (__m256bh) _mm256_loadu_si256(&y[0]));
+        accum256 = _mm256_dpbf16_ps(accum256, (__m256bh) _mm256_loadu_si256((__m256i *)&x[0]), (__m256bh) _mm256_loadu_si256((__m256i *)&y[0]));
         accum128 += _mm_add_ps(_mm256_castps256_ps128(accum256), _mm256_extractf128_ps(accum256, 1));
 
         /* Processing the remaining <16 chunk with 8-elements processing */
         if ((n&8) != 0) {
             int tail_index_16  = n&(~15);
-            accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128(&x[tail_index_16]), (__m128bh) _mm_loadu_si128(&y[tail_index_16]));
+            accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128((__m128i *)&x[tail_index_16]), (__m128bh) _mm_loadu_si128((__m128i *)&y[tail_index_16]));
         }
 
         /* Processing the remaining <8 chunk with masked 8-elements processing */
@@ -128,7 +128,7 @@ static float sbdot_accl_kernel(BLASLONG n, bfloat16 *x, bfloat16 *y)
         }
     } else if (n > 7) { /* n range from 8 to 15 */
         /* Processing <16 chunk with 8-elements processing */
-        accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128(&x[0]), (__m128bh) _mm_loadu_si128(&y[0]));
+        accum128 = _mm_dpbf16_ps(accum128, (__m128bh) _mm_loadu_si128((__m128i *)&x[0]), (__m128bh) _mm_loadu_si128((__m128i *)&y[0]));
 
         /* Processing the remaining <8 chunk with masked 8-elements processing */
         if ((n&7) != 0) {

kernel/x86_64/sbgemm_kernel_16x4_cooperlake.c

@@ -0,0 +1,499 @@
+/***************************************************************************
+Copyright (c) 2021, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <immintrin.h>
+#include "common.h"
+
+#define VMOVLDUP(addr, zmm) asm("vmovsldup (%1), %0": "=v"(zmm): "r"(addr))
+#define VMOVHDUP(addr, zmm) asm("vmovshdup (%1), %0": "=v"(zmm): "r"(addr))
+#define BROADCAST64(base, step, n, offset, zmm) \
+	if (n == 0) asm("vbroadcastsd %c2(%1), %0": "=v"(zmm): "r"(base), "n"(offset*2)); \
+	else asm("vbroadcastsd %c4(%1, %2, %c3), %0": "=v"(zmm): "r"(base), "r"(step), "n"(n*2), "n"(offset*2))
+
+#define DECLARE_A_PAIR(A) \
+	__m512i A_lo_##A; __m512i A_hi_##A;
+
+#define LOAD_A_PAIR(A) \
+	VMOVLDUP(ptr_a##A, A_lo_##A); \
+	VMOVHDUP(ptr_a##A, A_hi_##A);
+
+#define MASK_LOAD_A_PAIR(A) { \
+	__m512 tmp = _mm512_maskz_loadu_ps(mmask, ptr_a##A); \
+	A_lo_##A = (__m512i) _mm512_moveldup_ps(tmp); \
+	A_hi_##A = (__m512i) _mm512_movehdup_ps(tmp); \
+}
+
+#define LOAD_A_PAIR_TAIL(A) { \
+	__m256i ymm = _mm256_loadu_si256((void *)ptr_a##A); \
+	__m512 zmm = (__m512) _mm512_cvtepu16_epi32(ymm); \
+	A_lo_##A = (__m512i) _mm512_moveldup_ps(zmm); \
+	A_hi_##A = (__m512i) _mm512_movehdup_ps(zmm); \
+}
+
+#define MASK_LOAD_A_PAIR_TAIL(A) { \
+	__m256i ymm = _mm256_maskz_loadu_epi16(mmask, ptr_a##A); \
+	__m512 zmm = (__m512) _mm512_cvtepu16_epi32(ymm); \
+	A_lo_##A = (__m512i) _mm512_moveldup_ps(zmm); \
+	A_hi_##A = (__m512i) _mm512_movehdup_ps(zmm); \
+}
+
+#define DECLARE_B_PAIR() \
+	__m512i B_lo; __m512i B_hi;
+
+#define PREFETCH_B_STEP 32
+#define PREFETCH_B(Bx, By) \
+	if (By == 0) asm("prefetcht0 %c1(%0)": : "r"(ptr_b##Bx), "n"(PREFETCH_B_STEP * 2)); \
+	else asm("prefetcht0 %c3(%0, %1, %c2)": : "r"(ptr_b##Bx), "r"(n_blksize), "n"(By*2), "n"(PREFETCH_B_STEP * 2))
+
+#define BROADCAST_B_PAIR(Bx, By) \
+	BROADCAST64(ptr_b##Bx, n_blksize, By, 0, B_lo); \
+	BROADCAST64(ptr_b##Bx, n_blksize, By, 4, B_hi);
+
+#define MASK_BROADCAST_B_PAIR(Bx, x) {\
+	__m128 xmm = _mm_maskz_loadu_ps(nmask, ptr_b##Bx); \
+	B_lo = (__m512i) _mm512_broadcastsd_pd((__m128d) xmm); \
+	B_hi = (__m512i) _mm512_broadcastsd_pd(_mm_permute_pd((__m128d) xmm, 0x1)); \
+}
+
+#define BROADCAST_B_PAIR_TAIL(Bx, By) {\
+	__m128i xmm = (__m128i) _mm_load_sd((double *)(ptr_b##Bx + n_blksize * By)); \
+	xmm = _mm_cvtepu16_epi32(xmm); \
+	B_lo = _mm512_broadcast_i32x2(xmm); \
+	B_hi = _mm512_broadcast_i32x2((__m128i) _mm_permute_pd((__m128d) xmm, 0x1)); \
+}
+
+#define MASK_BROADCAST_B_PAIR_TAIL(Bx, By) {\
+	__m128i xmm = _mm_maskz_loadu_epi16(nmask, ptr_b##Bx + n_blksize * By); \
+	xmm = _mm_cvtepu16_epi32(xmm); \
+	B_lo = _mm512_broadcast_i32x2(xmm); \
+	B_hi = _mm512_broadcast_i32x2((__m128i) _mm_permute_pd((__m128d) xmm, 0x1)); \
+}
+
+#define DECLARE_RESULT_4X(A, Bx, By) \
+	__m512 result_00_##A##Bx##By = _mm512_setzero_ps(); \
+	__m512 result_01_##A##Bx##By = _mm512_setzero_ps(); \
+	__m512 result_10_##A##Bx##By = _mm512_setzero_ps(); \
+	__m512 result_11_##A##Bx##By = _mm512_setzero_ps();
+
+#define FMA(a, b, r) r = _mm512_dpbf16_ps(r, (__m512bh)a, (__m512bh)b)
+
+#define MATMUL_4X(A, Bx, By) \
+	FMA(A_lo_##A, B_lo, result_00_##A##Bx##By); \
+	FMA(A_hi_##A, B_lo, result_01_##A##Bx##By); \
+	FMA(A_lo_##A, B_hi, result_10_##A##Bx##By); \
+	FMA(A_hi_##A, B_hi, result_11_##A##Bx##By);
+
+#define _STORE_C_2nx16(addr, val0, val1) \
+	asm("vfmadd213ps (%1), %2, %0": "+v"(val0) : "r"(addr), "v"(alpha_512)); \
+	asm("vfmadd213ps (%1, %3, 4), %2, %0": "+v"(val1) : "r"(addr), "v"(alpha_512), "r"(ldc)); \
+	asm("vmovups %0, (%1)": : "v"(val0), "r"(addr)); \
+	asm("vmovups %0, (%1, %2, 4)": : "v"(val1), "r"(addr), "r"(ldc))
+
+#define _MASK_STORE_C_2nx16(addr, val0, val1) \
+	asm("vfmadd213ps (%1), %2, %0 %{%3%} ": "+v"(val0) : "r"(addr), "v"(alpha_512), "k"(mmask)); \
+	asm("vfmadd213ps (%1, %3, 4), %2, %0 %{%4%}": "+v"(val1) : "r"(addr), "v"(alpha_512), "r"(ldc), "k"(mmask)); \
+	asm("vmovups %0, (%1) %{%2%}": : "v"(val0), "r"(addr), "k"(mmask)); \
+	asm("vmovups %0, (%1, %2, 4) %{%3%}": : "v"(val1), "r"(addr), "r"(ldc), "k"(mmask))
+
+#define _REORDER_C_2X(result_0, result_1) { \
+	__m512 tmp0, tmp1; \
+	tmp0 = _mm512_unpacklo_ps(result_0, result_1); \
+	tmp1 = _mm512_unpackhi_ps(result_0, result_1); \
+	result_0 = (__m512) _mm512_unpacklo_pd((__m512d) tmp0, (__m512d) tmp1); \
+	result_1 = (__m512) _mm512_unpackhi_pd((__m512d) tmp0, (__m512d) tmp1); \
+}
+
+#define _STORE_2X(ptr_c, result_0, result_1) {\
+	_REORDER_C_2X(result_0, result_1) \
+	_STORE_C_2nx16(ptr_c, result_0, result_1); \
+	ptr_c += ldc * 2; \
+}
+
+#define _MASK_STORE_2X(ptr_c, result_0, result_1) {\
+	_REORDER_C_2X(result_0, result_1) \
+	_MASK_STORE_C_2nx16(ptr_c, result_0, result_1); \
+	ptr_c += ldc * 2; \
+}
+
+#define STORE_4X(A, Bx, By) { \
+	_STORE_2X(ptr_c##A, result_00_##A##Bx##By, result_01_##A##Bx##By); \
+	_STORE_2X(ptr_c##A, result_10_##A##Bx##By, result_11_##A##Bx##By); \
+}
+
+#define MASK_STORE_4X(A, Bx, By) { \
+	_MASK_STORE_2X(ptr_c##A, result_00_##A##Bx##By, result_01_##A##Bx##By); \
+	_MASK_STORE_2X(ptr_c##A, result_10_##A##Bx##By, result_11_##A##Bx##By); \
+}
+
+#define _STORE_C_16(addr, val0) \
+	asm("vfmadd213ps (%1), %2, %0": "+v"(val0) : "r"(addr), "v"(alpha_512)); \
+	asm("vmovups %0, (%1)": : "v"(val0), "r"(addr));
+
+#define _MASK_STORE_C_16(addr, val0) \
+	asm("vfmadd213ps (%1), %2, %0 %{%3%} ": "+v"(val0) : "r"(addr), "v"(alpha_512), "k"(mmask)); \
+	asm("vmovups %0, (%1) %{%2%}": : "v"(val0), "r"(addr), "k"(mmask));
+
+#define N_STORE_4X(A, Bx, By) { \
+	_REORDER_C_2X(result_00_##A##Bx##By, result_01_##A##Bx##By); \
+	_REORDER_C_2X(result_10_##A##Bx##By, result_11_##A##Bx##By); \
+	switch(n_count) { \
+		case 3: _STORE_C_16(ptr_c + ldc * 2, result_10_##A##Bx##By); \
+		case 2: _STORE_C_16(ptr_c + ldc * 1, result_01_##A##Bx##By); \
+		case 1: _STORE_C_16(ptr_c + ldc * 0, result_00_##A##Bx##By); \
+	} \
+	ptr_c##A += ldc * n_count; \
+}
+
+#define N_MASK_STORE_4X(A, Bx, By) { \
+	_REORDER_C_2X(result_00_##A##Bx##By, result_01_##A##Bx##By); \
+	_REORDER_C_2X(result_10_##A##Bx##By, result_11_##A##Bx##By); \
+	switch(n_count) { \
+		case 3: _MASK_STORE_C_16(ptr_c + ldc * 2, result_10_##A##Bx##By); \
+		case 2: _MASK_STORE_C_16(ptr_c + ldc * 1, result_01_##A##Bx##By); \
+		case 1: _MASK_STORE_C_16(ptr_c + ldc * 0, result_00_##A##Bx##By); \
+	} \
+	ptr_c##A += ldc * n_count; \
+}
+
+
+int CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc)
+{
+	IFLOAT *ptr_a = A, *ptr_b = B;
+	IFLOAT *ptr_b0, *ptr_b1;
+	IFLOAT *ptr_a0, *ptr_a1;
+	FLOAT *ptr_c = C;
+	FLOAT *ptr_c0, *ptr_c1;
+	BLASLONG n_count = n;
+	BLASLONG m_count, k_count;
+	BLASLONG n_blksize = 4 * k;
+	BLASLONG cn_offset = 0;
+	__m512 alpha_512 = _mm512_broadcastss_ps(_mm_load_ss(&alpha));
+
+	for (; n_count > 23; n_count -= 24) {
+		IFLOAT *ptr_b00 = ptr_b;
+		IFLOAT *ptr_b10 = ptr_b + n_blksize * 3;
+		ptr_a0 = ptr_a;
+		ptr_c = C + cn_offset * ldc;
+		m_count = m;
+		for (; m_count > 15; m_count -= 16) {
+			ptr_b0 = ptr_b00;
+			ptr_b1 = ptr_b10;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0); DECLARE_RESULT_4X(0, 0, 1); DECLARE_RESULT_4X(0, 0, 2);
+			DECLARE_RESULT_4X(0, 1, 0); DECLARE_RESULT_4X(0, 1, 1); DECLARE_RESULT_4X(0, 1, 2);
+			k_count = k;
+			for (; k_count > 3; k_count -=4) {
+				LOAD_A_PAIR(0);
+				_mm_prefetch(ptr_a0 + 128, _MM_HINT_T0);
+				ptr_a0 += 16 * 2;
+				BROADCAST_B_PAIR(0, 0); PREFETCH_B(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); PREFETCH_B(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); PREFETCH_B(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+				BROADCAST_B_PAIR(1, 0); PREFETCH_B(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR(1, 1); PREFETCH_B(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR(1, 2); PREFETCH_B(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4 * 2;
+
+				LOAD_A_PAIR(0);
+				_mm_prefetch(ptr_a0 + 128, _MM_HINT_T0);
+				ptr_a0 += 16 * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+				BROADCAST_B_PAIR(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4 * 2;
+			}
+			for (; k_count > 1; k_count -=2) {
+				LOAD_A_PAIR(0);
+				ptr_a0 += 16 * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+				BROADCAST_B_PAIR(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4 * 2;
+			}
+			if (k_count > 0) {
+				LOAD_A_PAIR_TAIL(0);
+				ptr_a0 += 16;
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR_TAIL(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR_TAIL(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4;
+				BROADCAST_B_PAIR_TAIL(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR_TAIL(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR_TAIL(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4;
+			}
+			ptr_c0 = ptr_c;
+			STORE_4X(0, 0, 0); STORE_4X(0, 0, 1); STORE_4X(0, 0, 2);
+			STORE_4X(0, 1, 0); STORE_4X(0, 1, 1); STORE_4X(0, 1, 2);
+			ptr_c += 16;
+		}
+		if (m_count > 0) {
+			__mmask16 mmask = (1UL << m_count) - 1;
+			ptr_b0 = ptr_b00;
+			ptr_b1 = ptr_b10;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0); DECLARE_RESULT_4X(0, 0, 1); DECLARE_RESULT_4X(0, 0, 2);
+			DECLARE_RESULT_4X(0, 1, 0); DECLARE_RESULT_4X(0, 1, 1); DECLARE_RESULT_4X(0, 1, 2);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				MASK_LOAD_A_PAIR(0);
+				ptr_a0 += m_count * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+				BROADCAST_B_PAIR(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4 * 2;
+			}
+			if (k_count > 0) {
+				MASK_LOAD_A_PAIR_TAIL(0);
+				ptr_a0 += m_count;
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR_TAIL(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR_TAIL(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4;
+				BROADCAST_B_PAIR_TAIL(1, 0); MATMUL_4X(0, 1, 0);
+				BROADCAST_B_PAIR_TAIL(1, 1); MATMUL_4X(0, 1, 1);
+				BROADCAST_B_PAIR_TAIL(1, 2); MATMUL_4X(0, 1, 2);
+				ptr_b1 += 4;
+			}
+			ptr_c0 = ptr_c;
+			MASK_STORE_4X(0, 0, 0); MASK_STORE_4X(0, 0, 1); MASK_STORE_4X(0, 0, 2);
+			MASK_STORE_4X(0, 1, 0); MASK_STORE_4X(0, 1, 1); MASK_STORE_4X(0, 1, 2);
+			ptr_c += m_count;
+		}
+		ptr_b += 24 * k;
+		cn_offset += 24;
+	}
+	for (; n_count > 11; n_count -= 12) {
+		IFLOAT *ptr_b00 = ptr_b;
+		ptr_a0 = ptr_a;
+		ptr_a1 = ptr_a + 16 * k;
+		ptr_c = C + cn_offset * ldc;
+		m_count = m;
+		for (; m_count > 31; m_count -= 32) {
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0); DECLARE_A_PAIR(1);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0); DECLARE_RESULT_4X(0, 0, 1); DECLARE_RESULT_4X(0, 0, 2);
+			DECLARE_RESULT_4X(1, 0, 0); DECLARE_RESULT_4X(1, 0, 1); DECLARE_RESULT_4X(1, 0, 2);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				LOAD_A_PAIR(0); LOAD_A_PAIR(1);
+				ptr_a0 += 16 * 2;
+				ptr_a1 += 16 * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0); MATMUL_4X(1, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1); MATMUL_4X(1, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2); MATMUL_4X(1, 0, 2);
+				ptr_b0 += 4 * 2;
+			}
+			if (k_count > 0) {
+				LOAD_A_PAIR_TAIL(0); LOAD_A_PAIR_TAIL(1);
+				ptr_a0 += 16;
+				ptr_a1 += 16;
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0); MATMUL_4X(1, 0, 0);
+				BROADCAST_B_PAIR_TAIL(0, 1); MATMUL_4X(0, 0, 1); MATMUL_4X(1, 0, 1);
+				BROADCAST_B_PAIR_TAIL(0, 2); MATMUL_4X(0, 0, 2); MATMUL_4X(1, 0, 2);
+				ptr_b0 += 4;
+			}
+			ptr_c0 = ptr_c;
+			ptr_c1 = ptr_c + 16;
+			STORE_4X(0, 0, 0); STORE_4X(1, 0, 0);
+			STORE_4X(0, 0, 1); STORE_4X(1, 0, 1);
+			STORE_4X(0, 0, 2); STORE_4X(1, 0, 2);
+			ptr_c += 16 * 2;
+			ptr_a0 = ptr_a1;
+			ptr_a1 = ptr_a0 + 16 * k;
+		}
+		for (; m_count > 15; m_count -= 16) {
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0); DECLARE_RESULT_4X(0, 0, 1); DECLARE_RESULT_4X(0, 0, 2);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				LOAD_A_PAIR(0);
+				ptr_a0 += 16 * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+			}
+			if (k_count > 0) {
+				LOAD_A_PAIR_TAIL(0);
+				ptr_a0 += 16;
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR_TAIL(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR_TAIL(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4;
+			}
+			ptr_c0 = ptr_c;
+			STORE_4X(0, 0, 0); STORE_4X(0, 0, 1); STORE_4X(0, 0, 2);
+			ptr_c += 16;
+		}
+		if (m_count > 0) {
+			__mmask16 mmask = (1UL << m_count) - 1;
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0); DECLARE_RESULT_4X(0, 0, 1); DECLARE_RESULT_4X(0, 0, 2);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				MASK_LOAD_A_PAIR(0);
+				ptr_a0 += m_count * 2;
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4 * 2;
+			}
+			if (k_count > 0) {
+				MASK_LOAD_A_PAIR_TAIL(0);
+				ptr_a0 += m_count;
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				BROADCAST_B_PAIR_TAIL(0, 1); MATMUL_4X(0, 0, 1);
+				BROADCAST_B_PAIR_TAIL(0, 2); MATMUL_4X(0, 0, 2);
+				ptr_b0 += 4;
+			}
+			ptr_c0 = ptr_c;
+			MASK_STORE_4X(0, 0, 0); MASK_STORE_4X(0, 0, 1); MASK_STORE_4X(0, 0, 2);
+			ptr_c += m_count;
+		}
+		ptr_b += 12 * k;
+		cn_offset += 12;
+	}
+	for (; n_count > 3; n_count -= 4) {
+		IFLOAT *ptr_b00 = ptr_b;
+		ptr_a0 = ptr_a;
+		ptr_c = C + cn_offset * ldc;
+		m_count = m;
+		for (; m_count > 15; m_count -= 16) {
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				LOAD_A_PAIR(0);
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += 4 * 2;
+				ptr_a0 += 16 * 2;
+			}
+			if (k_count > 0) {
+				LOAD_A_PAIR_TAIL(0);
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += 4;
+				ptr_a0 += 16;
+			}
+			ptr_c0 = ptr_c;
+			STORE_4X(0, 0, 0);
+			ptr_c += 16;
+		}
+		if (m_count > 0) {
+			__mmask16 mmask = (1UL << m_count) - 1;
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				MASK_LOAD_A_PAIR(0);
+				BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += 4 * 2;
+				ptr_a0 += m_count * 2;
+			}
+			if (k_count > 0) {
+				MASK_LOAD_A_PAIR_TAIL(0);
+				BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += 4;
+				ptr_a0 += m_count;
+			}
+			ptr_c0 = ptr_c;
+			MASK_STORE_4X(0, 0, 0);
+			ptr_c += m_count;
+		}
+		ptr_b += 4 * k;
+		cn_offset += 4;
+	}
+	if (n_count > 0) {
+		__mmask8 nmask = (1UL << n_count) - 1;
+		IFLOAT *ptr_b00 = ptr_b;
+		ptr_a0 = ptr_a;
+		ptr_c = C + cn_offset * ldc;
+		m_count = m;
+		for (; m_count > 15; m_count -= 16) {
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				LOAD_A_PAIR(0);
+				MASK_BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += n_count * 2;
+				ptr_a0 += 16 * 2;
+			}
+			if (k_count > 0) {
+				LOAD_A_PAIR_TAIL(0);
+				MASK_BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += n_count;
+				ptr_a0 += 16;
+			}
+			ptr_c0 = ptr_c;
+			N_STORE_4X(0, 0, 0);
+			ptr_c += 16;
+		}
+		if (m_count > 0) {
+			__mmask16 mmask = (1UL << m_count) - 1;
+			ptr_b0 = ptr_b00;
+			DECLARE_A_PAIR(0);
+			DECLARE_B_PAIR();
+			DECLARE_RESULT_4X(0, 0, 0);
+			for (k_count = k; k_count > 1; k_count -=2) {
+				MASK_LOAD_A_PAIR(0);
+				MASK_BROADCAST_B_PAIR(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += n_count * 2;
+				ptr_a0 += m_count * 2;
+			}
+			if (k_count > 0) {
+				MASK_LOAD_A_PAIR_TAIL(0);
+				MASK_BROADCAST_B_PAIR_TAIL(0, 0); MATMUL_4X(0, 0, 0);
+				ptr_b0 += n_count;
+				ptr_a0 += m_count;
+			}
+			ptr_c0 = ptr_c;
+			N_MASK_STORE_4X(0, 0, 0);
+			ptr_c += m_count;
+		}
+	}
+	return 0;
+}