Revert "[WIP,Testing] remove the lock around the thread shutdown function aga…"

This reverts commit ef6f97624b.

.cirrus.yml

@@ -58,8 +58,8 @@ task:
   - export VALID_ARCHS="i386 x86_64"
   - xcrun --sdk macosx --show-sdk-path
   - xcodebuild -version
-  - export CC=/Applications/Xcode_16.3.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
-  - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_16.3.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.4.sdk -arch x86_64"
+  - export CC=/Applications/Xcode_26.0.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+  - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_26.0.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX26.0.sdk -arch x86_64"
   - make TARGET=CORE2 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 RANLIB="ls -l"
   always:
     config_artifacts:
@@ -78,8 +78,8 @@ task:
   - export #PATH=/opt/homebrew/opt/llvm/bin:$PATH
   - export #LDFLAGS="-L/opt/homebrew/opt/llvm/lib"
   - export #CPPFLAGS="-I/opt/homebrew/opt/llvm/include" 
-  - export CC=/Applications/Xcode_16.3.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
-  - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_16.3.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS18.4.sdk -arch arm64 -miphoneos-version-min=10.0"
+  - export CC=/Applications/Xcode_26.0.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+  - export CFLAGS="-O2 -unwindlib=none -Wno-macro-redefined -isysroot /Applications/Xcode_26.0.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.0.sdk -arch arm64 -miphoneos-version-min=10.0"
   - xcrun --sdk iphoneos --show-sdk-path
   - ls -l /Applications
   - make TARGET=ARMV8 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1 CROSS=1
@@ -127,7 +127,7 @@ task:
 FreeBSD_task:
   name: FreeBSD-gcc
   freebsd_instance:
-    image_family: freebsd-14-2
+    image_family: freebsd-14-3
   install_script:
   - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc
   compile_script:
@@ -138,7 +138,7 @@ FreeBSD_task:
 FreeBSD_task:
   name: freebsd-gcc-ilp64
   freebsd_instance:
-    image_family: freebsd-14-2
+    image_family: freebsd-14-3
   install_script:
   - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc
   compile_script:
@@ -148,7 +148,7 @@ FreeBSD_task:
 FreeBSD_task:
   name: FreeBSD-clang-openmp
   freebsd_instance:
-    image_family: freebsd-14-2
+    image_family: freebsd-14-3
   install_script:
   - pkg update -f && pkg upgrade -y && pkg install -y gmake gcc 
   - ln -s /usr/local/lib/gcc13/libgfortran.so.5.0.0 /usr/lib/libgfortran.so

.github/workflows/dynamic_arch.yml

@@ -1,6 +1,6 @@
 name: continuous build
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -11,17 +11,24 @@ permissions:
 
 jobs:
   build:
-    if: "github.repository == 'OpenMathLib/OpenBLAS'"
+    if: "github.repository == 'OpenMathLib/OpenBLAS' || github.event_name == 'workflow_dispatch'"
     runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, ubuntu-24.04-arm]
+        cc: [gcc, clang, clang-21]
         fortran: [gfortran, flang]
         build: [cmake, make]
         exclude:
           - os: macos-latest
+            cc: gcc
+          - os: macos-latest
+            cc: clang-21
+          - os: macos-latest
+            fortran: flang
+          - os: ubuntu-24.04-arm
             fortran: flang
 
     steps:
@@ -42,10 +49,23 @@ jobs:
       - name: Install Dependencies
         run: |
           if [ "$RUNNER_OS" == "Linux" ]; then
+            cat << EOF | sudo tee -a /etc/apt/apt.conf.d/01norecommend
+          APT::Install-Recommends "0";
+          APT::Install-Suggests "0";
+          EOF
             sudo apt-get update
-            sudo apt-get install -y gfortran cmake ccache
-            wget http://security.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
-            sudo apt install ./libtinfo5_6.3-2ubuntu0.1_amd64.deb
+            sudo apt-get install -y ccache
+            if [ "${{ matrix.cc }}" == "clang-21" ]; then
+                wget https://apt.llvm.org/llvm.sh
+                chmod +x llvm.sh
+                sudo ./llvm.sh 21
+            fi
+            if [ "${{ matrix.fortran }}" == "flang" ]; then
+              wget http://security.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
+              sudo apt install ./libtinfo5_6.3-2ubuntu0.1_amd64.deb
+            else
+              sudo apt-get install -y ${{ matrix.fortran }}
+            fi
           elif [ "$RUNNER_OS" == "macOS" ]; then
             # It looks like "gfortran" isn't working correctly unless "gcc" is re-installed.
             brew reinstall gcc
@@ -64,12 +84,12 @@ jobs:
           # GNU make and cmake call the compilers differently. It looks like
           # that causes the cache to mismatch. Keep the ccache for both build
           # tools separate to avoid polluting each other.
-          key: ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }}-${{ github.ref }}-${{ github.sha }}
+          key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}-${{ matrix.cc }}-${{ matrix.fortran }}-${{ github.ref }}-${{ github.sha }}
           # Restore a matching ccache cache entry. Prefer same branch and same Fortran compiler.
           restore-keys: |
-            ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }}-${{ github.ref }}
-            ccache-${{ runner.os }}-${{ matrix.build }}-${{ matrix.fortran }}
-            ccache-${{ runner.os }}-${{ matrix.build }}
+            ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}-${{ matrix.cc }}-${{ matrix.fortran }}-${{ github.ref }}
+            ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}-${{ matrix.cc }}-${{ matrix.fortran }}
+            ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build }}-${{ matrix.cc }}
 
       - name: Configure ccache
         run: |
@@ -89,7 +109,7 @@ jobs:
           echo "max_size = 300M" > ~/.ccache/ccache.conf
           echo "compression = true" >> ~/.ccache/ccache.conf
           ccache -s
-          
+
       - name: Add gfortran runtime to link path
         if: matrix.build == 'make' && runner.os == 'macOS'
         run: |
@@ -110,7 +130,7 @@ jobs:
           fi
           case "${{ matrix.build }}" in
             "make")
-              make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}"
+              make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 CC="ccache ${{ matrix.cc }}" FC="ccache ${{ matrix.fortran }}"
               ;;
             "cmake")
               mkdir build && cd build
@@ -119,6 +139,7 @@ jobs:
                     -DBUILD_WITHOUT_LAPACK=0 \
                     -DCMAKE_VERBOSE_MAKEFILE=ON \
                     -DCMAKE_BUILD_TYPE=Release \
+                    -DCMAKE_C_COMPILER=${{ matrix.cc }} \
                     -DCMAKE_Fortran_COMPILER=${{ matrix.fortran }} \
                     -DCMAKE_C_COMPILER_LAUNCHER=ccache \
                     -DCMAKE_Fortran_COMPILER_LAUNCHER=ccache \
@@ -142,13 +163,13 @@ jobs:
             "make")
               MAKE_FLAGS='DYNAMIC_ARCH=1 USE_OPENMP=0'
               echo "::group::Tests in 'test' directory"
-              make -C test $MAKE_FLAGS FC="ccache ${{ matrix.fortran }}"
+              make -C test $MAKE_FLAGS CC="ccache ${{ matrix.cc }}" FC="ccache ${{ matrix.fortran }}"
               echo "::endgroup::"
               echo "::group::Tests in 'ctest' directory"
-              make -C ctest $MAKE_FLAGS FC="ccache ${{ matrix.fortran }}"
+              make -C ctest $MAKE_FLAGS CC="ccache ${{ matrix.cc }}" FC="ccache ${{ matrix.fortran }}"
               echo "::endgroup::"
               echo "::group::Tests in 'utest' directory"
-              make -C utest $MAKE_FLAGS FC="ccache ${{ matrix.fortran }}"
+              make -C utest $MAKE_FLAGS CC="ccache ${{ matrix.cc }}" FC="ccache ${{ matrix.fortran }}"
               echo "::endgroup::"
               ;;
             "cmake")
@@ -372,15 +393,15 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-        
+
       - name: Install Dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y gcc gfortran make
-          
+
       - name: Build OpenBLAS
         run: |
-          make -j${nproc} 
+          make -j${nproc}
           make -j${nproc} lapack-test
-    
-          
+
+

.github/workflows/riscv64_vector.yml

@@ -16,7 +16,7 @@ jobs:
     env:
       triple: riscv64-unknown-linux-gnu
       riscv_gnu_toolchain: https://github.com/riscv-collab/riscv-gnu-toolchain
-      riscv_gnu_toolchain_version: 13.2.0
+      riscv_gnu_toolchain_version: 15.1.0
       riscv_gnu_toolchain_nightly_download_path: /releases/download/2025.08.29/riscv64-glibc-ubuntu-22.04-llvm-nightly-2025.08.29-nightly.tar.xz
     strategy:
       fail-fast: false
@@ -26,8 +26,8 @@ jobs:
             opts: TARGET=RISCV64_ZVL128B BINARY=64 ARCH=riscv64
             qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=128,elen=64
           - target: RISCV64_ZVL256B
-            opts: TARGET=RISCV64_ZVL256B BINARY=64 ARCH=riscv64
-            qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64
+            opts: TARGET=RISCV64_ZVL256B BINARY=64 ARCH=riscv64 BUILD_BFLOAT16=1 BUILD_HFLOAT16=1
+            qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64,zfh=true,zvfh=true,zvfbfwma=true
           - target: DYNAMIC_ARCH=1
             opts: TARGET=RISCV64_GENERIC BINARY=64 ARCH=riscv64 DYNAMIC_ARCH=1
             qemu_cpu: rv64,g=true,c=true,v=true,vext_spec=v1.0,vlen=256,elen=64
@@ -40,10 +40,13 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install autoconf automake autotools-dev ninja-build make \
-          libgomp1-riscv64-cross ccache
+          libgomp1-riscv64-cross ccache qemu-kvm qemu-user libc6-riscv64-cross
           wget ${riscv_gnu_toolchain}/${riscv_gnu_toolchain_nightly_download_path}
           tar -xvf $(basename ${riscv_gnu_toolchain_nightly_download_path}) -C /opt
-
+          wget https://gist.github.com/martin-frbg/bb630e0de34978e578eeb496b1538d4e/raw/7fd8d971f327f7a517b8f5f7989479ff2b36f71f/qemu-riscv64-10.1-ubuntu24 -P /opt/riscv/bin -o riscv64-qemu
+          mv /opt/riscv/bin/qemu-riscv64-10.1-ubuntu24 /opt/riscv/bin/qemu-riscv64
+          chmod +x /opt/riscv/bin/qemu-riscv64
+          
       - name: Compilation cache
         uses: actions/cache@v3
         with:
@@ -74,7 +77,7 @@ jobs:
         run: |
             export PATH="/opt/riscv/bin:$PATH"
             make TARGET=${{ matrix.target }} CFLAGS="-DTARGET=${{ matrix.target }}" \
-            CC='${triple}-gcc' \
+            CC='ccache clang --rtlib=compiler-rt -target ${triple} --sysroot /opt/riscv/sysroot --gcc-toolchain=/opt/riscv/lib/gcc/riscv64-unknown-linux-gnu/${riscv_gnu_toolchain_version}/' \
             AR='ccache ${triple}-ar' AS='ccache ${triple}-gcc' LD='ccache ${triple}-gcc' \
             RANLIB='ccache ${triple}-ranlib' \
             FC='ccache ${triple}-gfortran' ${{ matrix.opts }} \
@@ -98,6 +101,8 @@ jobs:
         shell: bash
         run: |
           export PATH="/opt/riscv/bin:$PATH"
+          export LD_LIBRARY_PATH=/opt/riscv/sysroot/lib
+          sudo ln -s /opt/riscv/sysroot/lib/ld-linux-riscv64-lp64d.so.1 /lib
           export QEMU_CPU=${{ matrix.qemu_cpu }}
           rm -rf ./test_out
           mkdir -p ./test_out
@@ -134,6 +139,14 @@ jobs:
           wait
           while IFS= read -r -d $'\0' LOG; do cat $LOG ; FAILURES=1 ; done < <(grep -lZ FAIL ./test_out/*)
           if [[ ! -z $FAILURES ]]; then echo "==========" ; echo "== FAIL ==" ; echo "==========" ; echo ; exit 1 ; fi
+          if [ "${{matrix.target}}" == "RISCV64_ZVL256B" ]; then
+          qemu-riscv64 test/test_sbgemm &
+          qemu-riscv64 test/test_sbgemv &
+          qemu-riscv64 test/test_shgemm &
+          qemu-riscv64 test/test_shgemv &
+          qemu-riscv64 test/test_bgemm
+          fi
+
 
       - name: netlib tests
         shell: bash

.gitignore

@@ -80,6 +80,7 @@ test/SBLAT3_3M.SUMM
 test/ZBLAT2.SUMM
 test/ZBLAT3.SUMM
 test/ZBLAT3_3M.SUMM
+test/SHBLAT2.SUMM
 test/SHBLAT3.SUMM
 test/SBBLAT2.SUMM
 test/SBBLAT3.SUMM
@@ -98,6 +99,7 @@ test/sblat2
 test/sblat3
 test/sblat3_3m
 test/test_shgemm
+test/test_shgemv
 test/test_sbgemm
 test/test_sbgemv
 test/test_bgemm

CMakeLists.txt

@@ -498,6 +498,11 @@ if (BUILD_SHARED_LIBS OR DELETE_STATIC_LIBS AND NOT ${SYMBOLPREFIX}${SYMBOLSUFFI
   else ()
 	  set (BBF16 0)
   endif()
+  if (${BUILD_HFLOAT16})
+	  set (BHF16 1)
+  else ()
+	  set (BHF16 0)
+  endif()
   if (${BUILD_SINGLE})
 	  set (BS 1)
   else ()
@@ -533,7 +538,7 @@ endif()
  #if (USE_PERL)
 message(STATUS "adding postbuild instruction to rename syms")
   add_custom_command(TARGET ${OpenBLAS_LIBNAME}_static POST_BUILD
-	COMMAND perl ${PROJECT_SOURCE_DIR}/exports/gensymbol.pl "win2k" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" "${SYMBOLPREFIX}" "${SYMBOLSUFFIX}" "${BLD}" "${BBF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/renamesyms.def
+	COMMAND perl ${PROJECT_SOURCE_DIR}/exports/gensymbol.pl "win2k" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" "${SYMBOLPREFIX}" "${SYMBOLSUFFIX}" "${BLD}" "${BBF16}" "${BHF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/renamesyms.def
 	COMMAND ${CMAKE_C_COMPILER} ${CMAKE_C_FLAGS} -I${PROJECT_SOURCE_DIR} -I${PROJECT_BINARY_DIR} -c -o ${PROJECT_BINARY_DIR}/dllinit.o ${PROJECT_SOURCE_DIR}/exports/dllinit.c
 	COMMAND lld-link -nodefaultlib:libcmt -defaultlib:${CRTLIB} ${CMAKE_LINKER_FLAGS} -errorlimit:0 -def:${PROJECT_BINARY_DIR}/renamesyms.def ${PROJECT_BINARY_DIR}/dllinit.o $<TARGET_FILE:${OpenBLAS_LIBNAME}_static> -wholearchive:$<TARGET_FILE:${OpenBLAS_LIBNAME}_static> -dll -out:$<TARGET_FILE_DIR:${OpenBLAS_LIBNAME}_static>/${OpenBLAS_LIBNAME}.dll -implib:$<TARGET_FILE_DIR:${OpenBLAS_LIBNAME}_static>/${OpenBLAS_LIBNAME}.dll.a ${PDBOPT} 
 	#COMMAND lld-link -nodefaultlib:libcmt -defaultlib:msvcrt ${CMAKE_LINKER_FLAGS} -errorlimit:0 -def:${PROJECT_BINARY_DIR}/renamesyms.def ${PROJECT_BINARY_DIR}/dllinit.o $<TARGET_FILE:${OpenBLAS_LIBNAME}_static> -wholearchive:$<TARGET_FILE:${OpenBLAS_LIBNAME}_static> -dll -out:$<TARGET_FILE_DIR:${OpenBLAS_LIBNAME}_static>/${OpenBLAS_LIBNAME}.dll -implib:$<TARGET_FILE_DIR:${OpenBLAS_LIBNAME}_static>/${OpenBLAS_LIBNAME}.dll.a
@@ -543,13 +548,13 @@ message(STATUS "adding postbuild instruction to rename syms")
   else ()
   if (NOT USE_PERL)
   add_custom_command(TARGET ${OpenBLAS_LIBNAME}_shared POST_BUILD
-	  COMMAND sh ${PROJECT_SOURCE_DIR}/exports/gensymbol "objcopy" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" \"${SYMBOLPREFIX}\" \"${SYMBOLSUFFIX}\" "${BLD}" "${BBF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/objcopy.def
+	  COMMAND sh ${PROJECT_SOURCE_DIR}/exports/gensymbol "objcopy" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" \"${SYMBOLPREFIX}\" \"${SYMBOLSUFFIX}\" "${BLD}" "${BBF16}" "${BHF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/objcopy.def
     COMMAND objcopy --redefine-syms ${PROJECT_BINARY_DIR}/objcopy.def  ${PROJECT_BINARY_DIR}/lib/${OpenBLAS_LIBNAME}.so
     COMMENT "renaming symbols"
     )
   else()
   add_custom_command(TARGET ${OpenBLAS_LIBNAME}_shared POST_BUILD
-    COMMAND perl ${PROJECT_SOURCE_DIR}/exports/gensymbol.pl "objcopy" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" \"${SYMBOLPREFIX}\" \"${SYMBOLSUFFIX}\" "${BLD}" "${BBF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/objcopy.def
+    COMMAND perl ${PROJECT_SOURCE_DIR}/exports/gensymbol.pl "objcopy" "${ARCH}" "${BU}" "${EXPRECISION_IN}" "${NO_CBLAS_IN}" "${NO_LAPACK_IN}" "${NO_LAPACKE_IN}" "${NEED2UNDERSCORES_IN}" "${ONLY_CBLAS_IN}" \"${SYMBOLPREFIX}\" \"${SYMBOLSUFFIX}\" "${BLD}" "${BBF16}" "${BHF16}" "${BS}" "${BD}" "${BC}" "${BZ}" > ${PROJECT_BINARY_DIR}/objcopy.def
     COMMAND objcopy --redefine-syms ${PROJECT_BINARY_DIR}/objcopy.def  ${PROJECT_BINARY_DIR}/lib/lib${OpenBLAS_LIBNAME}.so
     COMMENT "renaming symbols"
     )

Makefile.power

@@ -118,6 +118,10 @@ ifeq ($(F_COMPILER), GFORTRAN)
 FCOMMON_OPT += -fno-optimize-sibling-calls
 endif
 
+ifeq ($(F_COMPILER), FLANGNEW)
+override FCOMMON_OPT := $(filter-out -frecursive,$(FCOMMON_OPT))
+endif
+
 FLAMEPATH	= $(HOME)/flame/lib
 
 #ifeq ($(CORE), CELL)
@@ -147,12 +151,15 @@ endif
 ifdef BINARY64
 
 
+#Skip C/Fortran compiler combination checks in AIX if NO_FORTRAN or ONLY_CBLAS is set
+ifeq ($(filter 1,$(NO_FORTRAN) $(ONLY_CBLAS)),)
 ifeq ($(C_COMPILER)$(F_COMPILER)$(OSNAME), GCCIBMAIX)
 $(error Using GCC and XLF on AIX is not a supported combination.)
 endif
 ifeq ($(C_COMPILER)$(F_COMPILER)$(OSNAME), CLANGGFORTRANAIX)
 $(error Using Clang and gFortran on AIX is not a supported combination.)
 endif
+endif
 
 ifeq ($(OSNAME), AIX)
 ifeq ($(C_COMPILER), GCC)

Makefile.riscv64

@@ -6,23 +6,20 @@ ifeq ($(CORE), x280)
 CCOMMON_OPT += -march=rv64imafdcv_zba_zbb_zfh_zvl512b -mabi=lp64d 
 FCOMMON_OPT += -march=rv64imafdcv_zba_zbb_zfh -mabi=lp64d -static
 endif
-ifeq ($(CORE), RISCV64_ZVL256B)
+RISCV64_OPT  = rv64imafdcv
 ifeq ($(BUILD_HFLOAT16), 1)
-CCOMMON_OPT += -march=rv64imafdcv_zvl256b_zvfh_zfh -mabi=lp64d
-FCOMMON_OPT += -march=rv64imafdcv_zvfh_zfh -mabi=lp64d
-else
-CCOMMON_OPT += -march=rv64imafdcv_zvl256b -mabi=lp64d
-FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d
+RISCV64_OPT := $(RISCV64_OPT)_zvfh_zfh
 endif
+ifeq ($(BUILD_BFLOAT16), 1)
+RISCV64_OPT := $(RISCV64_OPT)_zvfbfwma
+endif
+ifeq ($(CORE), RISCV64_ZVL256B)
+CCOMMON_OPT += -march=$(RISCV64_OPT)_zvl256b -mabi=lp64d
+FCOMMON_OPT += -march=$(RISCV64_OPT) -mabi=lp64d
 endif
 ifeq ($(CORE), RISCV64_ZVL128B)
-ifeq ($(BUILD_HFLOAT16), 1)
-CCOMMON_OPT += -march=rv64imafdcv_zvfh_zfh -mabi=lp64d
-FCOMMON_OPT += -march=rv64imafdcv_zvfh_zfh -mabi=lp64d
-else
-CCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d
-FCOMMON_OPT += -march=rv64imafdcv -mabi=lp64d
-endif
+CCOMMON_OPT += -march=$(RISCV64_OPT) -mabi=lp64d
+FCOMMON_OPT += -march=$(RISCV64_OPT) -mabi=lp64d
 endif
 ifeq ($(CORE), RISCV64_GENERIC)
 CCOMMON_OPT += -march=rv64imafdc -mabi=lp64d

Makefile.system

@@ -441,7 +441,7 @@ ifeq (x$(XCVER), x 15)
 CCOMMON_OPT += -Wl,-ld_classic
 FCOMMON_OPT += -Wl,-ld_classic
 endif
-ifeq (x$(XCVER), x 16)
+ifeq ($(shell [ $(XCVER) -ge 16 ] && echo yes),yes)
 override CEXTRALIB := $(filter-out(-lto_library, $(CEXTRALIB))) 
 endif
 endif
@@ -887,7 +887,7 @@ NO_BINARY_MODE  = 1
 BINARY_DEFINED  = 1
 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
-ifeq ($(F_COMPILER), GFORTRAN)
+ifeq ($(F_COMPILER), $(filter $(F_COMPILER),GFORTRAN FLANGNEW))
 FCOMMON_OPT +=  -fdefault-integer-8
 endif
 ifeq ($(F_COMPILER), FLANG)
@@ -902,7 +902,7 @@ NO_BINARY_MODE  = 1
 BINARY_DEFINED  = 1
 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
-ifeq ($(F_COMPILER), GFORTRAN)
+ifeq ($(F_COMPILER), $(filter $(F_COMPILER),GFORTRAN FLANGNEW))
 FCOMMON_OPT +=  -fdefault-integer-8
 endif
 ifeq ($(F_COMPILER), FLANG)
@@ -917,7 +917,7 @@ NO_BINARY_MODE  = 1
 BINARY_DEFINED  = 1
 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
-ifeq ($(F_COMPILER), GFORTRAN)
+ifeq ($(F_COMPILER), $(filter $(F_COMPILER),GFORTRAN FLANGNEW))
 FCOMMON_OPT +=  -fdefault-integer-8
 endif
 ifeq ($(F_COMPILER), FLANG)

azure-pipelines.yml

@@ -95,7 +95,7 @@ jobs:
      vmImage: 'windows-latest'
   steps:   
   - script: |
-      mingw32-make CC=gcc FC=gfortran DYNAMIC_ARCH=1 DYNAMIC_LIST="SANDYBRIDGE"
+      mingw32-make CC=gcc NOLAPACK=1 DYNAMIC_ARCH=1 DYNAMIC_LIST="SANDYBRIDGE"
 
 - job: Windows_clang_cmake
   pool:
@@ -155,7 +155,7 @@ jobs:
 
 - job: OSX_OpenMP
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   steps:   
   - script: |
       brew update
@@ -165,19 +165,19 @@ jobs:
      
 - job: OSX_GCC_Nothreads
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   steps:   
   - script: |
       brew update
       make USE_THREADS=0 CC=gcc-13 FC=gfortran-13
      
-- job: OSX_GCC12
+- job: OSX_GCC15
   pool:
      vmImage: 'macOS-latest'
   steps:   
   - script: |
       brew update
-      make CC=gcc-12 FC=gfortran-12
+      make CC=gcc-15 FC=gfortran-15
 
 - job: OSX_LLVM_flangnew
   pool:
@@ -201,7 +201,7 @@ jobs:
   - script: |
       brew update
       brew install llvm libomp
-      make TARGET=CORE2 USE_OPENMP=1 DYNAMIC_ARCH=1 CC=/usr/local/opt/llvm/bin/clang NOFORTRAN=1
+      make TARGET=CORE2 USE_OPENMP=1 DYNAMIC_ARCH=1 DYNAMIC_LIST='NEHALEM HASWELL SKYLAKEX' CC=/usr/local/opt/llvm/bin/clang NOFORTRAN=1
 
 - job: OSX_OpenMP_Clang_cmake
   pool:
@@ -215,13 +215,13 @@ jobs:
       brew install llvm libomp
       mkdir build
       cd build
-      cmake -DTARGET=CORE2 -DUSE_OPENMP=1 -DINTERFACE64=1 -DDYNAMIC_ARCH=1 -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang -DNOFORTRAN=1 -DNO_AVX512=1 ..
+      cmake -DTARGET=CORE2 -DUSE_OPENMP=1 -DINTERFACE64=1 -DDYNAMIC_ARCH=1 -DDYNAMIC_LIST='NEHALEM HASWELL SKYLAKEX' -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang -DNOFORTRAN=1 -DNO_AVX512=1 ..
       make
       ctest
       
 - job: OSX_dynarch_cmake
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   variables:
      LD_LIBRARY_PATH: /usr/local/opt/llvm/lib
      LIBRARY_PATH: /usr/local/opt/llvm/lib
@@ -268,7 +268,7 @@ jobs:
  
 - job: OSX_NDK_ARMV7
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   steps:   
   - script: | 
       brew update
@@ -278,35 +278,37 @@ jobs:
 
 - job: OSX_IOS_ARMV8
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   variables:
-     CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
-     CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS16.2.sdk -arch arm64 -miphoneos-version-min=10.0
+     CC: /Applications/Xcode_16.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+     CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_16.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS18.2.sdk -arch arm64 -miphoneos-version-min=10.0
   steps:
   - script: |
+     ls /Applications/Xcode_16.2.app/Contents/Developer/Platforms/
+     ls /Applications/Xcode_16.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs
      make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1
 
 - job: OSX_IOS_ARMV7
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   variables:
-     CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
-     CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS16.2.sdk -arch armv7 -miphoneos-version-min=5.1
+     CC: /Applications/Xcode_16.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+     CFLAGS: -O2 -mno-thumb -Wno-macro-redefined -isysroot /Applications/Xcode_16.2.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS18.2.sdk -arch armv7 -miphoneos-version-min=5.1
   steps:
   - script: |
      make TARGET=ARMV7 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1
 
 - job: OSX_xbuild_DYNAMIC_ARM64
   pool:
-     vmImage: 'macOS-13'
+     vmImage: 'macOS-14'
   variables:
-     CC: /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
-     CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.1.sdk -arch arm64
+     CC: /Applications/Xcode_16.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+     CFLAGS: -O2 -Wno-macro-redefined -isysroot /Applications/Xcode_16.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.2.sdk -arch arm64
   steps:
   - script: |
-     ls /Applications/Xcode_14.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs
+     ls /Applications/Xcode_16.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs
      /Applications/Xcode_12.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -arch arm64 --print-supported-cpus
-     /Applications/Xcode_14.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang --version
+     /Applications/Xcode_16.2.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang --version
      make TARGET=ARMV8 DYNAMIC_ARCH=1 NUM_THREADS=32 HOSTCC=clang NOFORTRAN=1
 
 - job: ALPINE_MUSL

cblas.h

@@ -456,6 +456,14 @@ void cblas_cgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enu
 void cblas_zgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array,
 		       OPENBLAS_CONST void * alpha_array, OPENBLAS_CONST void ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST void ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST void * beta_array, void ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size);
 
+void cblas_sgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST float * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST float * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST float beta, float * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size);
+
+void cblas_dgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST double alpha, OPENBLAS_CONST double * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST double * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST double beta, double * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size);
+
+void cblas_cgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST void * alpha, OPENBLAS_CONST void * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST void * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST void * beta, void * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size);
+
+void cblas_zgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST void * alpha, OPENBLAS_CONST void * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST void * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST void * beta, void * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size);
+
 /*** BFLOAT16 and INT8 extensions ***/
 /* convert float array to BFLOAT16 array by rounding */
 void   cblas_sbstobf16(OPENBLAS_CONST blasint n, OPENBLAS_CONST float  *in, OPENBLAS_CONST blasint incin, bfloat16 *out, OPENBLAS_CONST blasint incout);
@@ -477,6 +485,7 @@ void   cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum C
 void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array,
 		       OPENBLAS_CONST float * alpha_array, OPENBLAS_CONST bfloat16 ** A_array, OPENBLAS_CONST blasint * lda_array, OPENBLAS_CONST bfloat16 ** B_array, OPENBLAS_CONST blasint * ldb_array, OPENBLAS_CONST float * beta_array, float ** C_array, OPENBLAS_CONST blasint * ldc_array, OPENBLAS_CONST blasint group_count, OPENBLAS_CONST blasint * group_size);
 
+void cblas_sbgemm_batch_strided(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 * A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST blasint stridea, OPENBLAS_CONST bfloat16 * B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST blasint strideb, OPENBLAS_CONST float beta, float * C, OPENBLAS_CONST blasint ldc, OPENBLAS_CONST blasint stridec, OPENBLAS_CONST blasint group_size);
 /*** FLOAT16 extensions ***/
 void cblas_shgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
 		    OPENBLAS_CONST float alpha, OPENBLAS_CONST hfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST hfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc);

cmake/OpenBLASConfig.cmake.in

@@ -50,30 +50,30 @@ set(PN OpenBLAS)
 
 # need to check that the @USE_*@ evaluate to something cmake can perform boolean logic upon
 if(@USE_OPENMP@)
-    set(${PN}_openmp_FOUND 1)
+    set(${PN}@SUFFIX64@_openmp_FOUND 1)
 elseif(@USE_THREAD@)
-    set(${PN}_pthread_FOUND 1)
+    set(${PN}@SUFFIX64@_pthread_FOUND 1)
 else()
-    set(${PN}_serial_FOUND 1)
+    set(${PN}@SUFFIX64@_serial_FOUND 1)
 endif()
 
-check_required_components(${PN})
+check_required_components(${PN}@SUFFIX64@)
 
 #-----------------------------------------------------------------------------
 # Don't include targets if this file is being picked up by another
 # project which has already built this as a subproject
 #-----------------------------------------------------------------------------
-if(NOT TARGET ${PN}::OpenBLAS)
-    include("${CMAKE_CURRENT_LIST_DIR}/${PN}Targets.cmake")
+if(NOT TARGET ${PN}@SUFFIX64@::OpenBLAS)
+    include("${CMAKE_CURRENT_LIST_DIR}/${PN}@SUFFIX64@Targets.cmake")
 
-    get_property(_loc TARGET ${PN}::OpenBLAS PROPERTY LOCATION)
-    set(${PN}_LIBRARY ${_loc})
-    get_property(_ill TARGET ${PN}::OpenBLAS PROPERTY INTERFACE_LINK_LIBRARIES)
-    set(${PN}_LIBRARIES ${_ill})
+    get_property(_loc TARGET ${PN}@SUFFIX64@::OpenBLAS PROPERTY LOCATION)
+    set(${PN}@SUFFIX64@_LIBRARY ${_loc})
+    get_property(_ill TARGET ${PN}@SUFFIX64@::OpenBLAS PROPERTY INTERFACE_LINK_LIBRARIES)
+    set(${PN}@SUFFIX64@_LIBRARIES ${_ill})
 
-    get_property(_id TARGET ${PN}::OpenBLAS PROPERTY INCLUDE_DIRECTORIES)
-    set(${PN}_INCLUDE_DIR ${_id})
-    get_property(_iid TARGET ${PN}::OpenBLAS PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
-    set(${PN}_INCLUDE_DIRS ${_iid})
+    get_property(_id TARGET ${PN}@SUFFIX64@::OpenBLAS PROPERTY INCLUDE_DIRECTORIES)
+    set(${PN}@SUFFIX64@_INCLUDE_DIR ${_id})
+    get_property(_iid TARGET ${PN}@SUFFIX64@::OpenBLAS PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
+    set(${PN}@SUFFIX64@_INCLUDE_DIRS ${_iid})
 endif()
 

cmake/cc.cmake

@@ -213,7 +213,7 @@ endif ()
 
 if (${CORE} STREQUAL A64FX)
   if (NOT DYNAMIC_ARCH)
-    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=a64fx")
     elseif (${GCC_VERSION} VERSION_GREATER 11.0 OR ${GCC_VERSION} VERSION_EQUAL 11.0)
       set (CCOMMON_OPT  "${CCOMMON_OPT} -march=armv8.2-a+sve -mtune=a64fx")
@@ -227,7 +227,7 @@ if (${CORE} STREQUAL NEOVERSEV2)
 	if (NOT DYNAMIC_ARCH)
 		if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
 			set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-v2")
-		elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+		elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 			set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-v2")
 		else ()
 			if (${GCC_VERSION} VERSION_GREATER 13.0 OR ${GCC_VERSION} VERSION_EQUAL 13.0)
@@ -245,7 +245,7 @@ if (${CORE} STREQUAL NEOVERSEN2)
   if (NOT DYNAMIC_ARCH)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2")
-    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-v2")
     else ()
       if (${GCC_VERSION} VERSION_GREATER 11.1 OR ${GCC_VERSION} VERSION_EQUAL 11.1)
@@ -261,7 +261,7 @@ if (${CORE} STREQUAL NEOVERSEV1)
   if (NOT DYNAMIC_ARCH)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
-    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-v1")
     else ()
       if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4)
@@ -275,7 +275,7 @@ endif ()
 
 if (${CORE} STREQUAL NEOVERSEN1)
   if (NOT DYNAMIC_ARCH)
-    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-n1")
     elseif (${GCC_VERSION} VERSION_GREATER 9.4 OR ${GCC_VERSION} VERSION_EQUAL 9.4)
       set (CCOMMON_OPT  "${CCOMMON_OPT} -march=armv8.2-a -mtune=neoverse-n1")
@@ -287,7 +287,7 @@ endif ()
 
 if (${CORE} STREQUAL AMPEREONE)
   if (NOT DYNAMIC_ARCH)
-    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVC")
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC")
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-n1")
     elseif (${GCC_VERSION} VERSION_GREATER 12.1)
       set (CCOMMON_OPT  "${CCOMMON_OPT} -march=armv8.6-a+crypto+crc+fp16+sha3+rng -mtune=ampereone")
@@ -301,7 +301,7 @@ if (${CORE} STREQUAL ARMV8SVE)
   if (NOT DYNAMIC_ARCH)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8-a+sve")
-    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	    set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=host")
     else ()
       set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8-a+sve")
@@ -311,7 +311,7 @@ endif ()
 
 if (${CORE} STREQUAL ARMV9SME)
   if (NOT DYNAMIC_ARCH)
-    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
 	    set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=host")
     else ()
     set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv9-a+sme")
@@ -391,6 +391,30 @@ if (${CORE} STREQUAL PPCG4)
   endif ()
 endif ()
 
+
+if ((${CORE} STREQUAL RISCV64_ZVL128B) OR (${CORE} STREQUAL RISCV64_ZVL256B))
+  set (RISCV64_OPT "rv64imafdcv")
+  if (BUILD_BFLOAT16)
+    set (RISCV64_OPT "${RISCV64_OPT}_zvfbfwma")
+  endif()
+  if (BUILD_HFLOAT16)
+    set (RISCV64_OPT "${RISCV64_OPT}_zvfh_zfh")
+  endif()
+  if (${CORE} STREQUAL RISCV64_ZVL256B)
+    set (CCOMMON_OPT "${CCOMMON_OPT} -march=${RISCV64_OPT}_zvl256b -mabi=lp64d")
+  endif()
+  if (${CORE} STREQUAL RISCV64_ZVL128B)
+    set (CCOMMON_OPT "${CCOMMON_OPT} -march=${RISCV64_OPT}_zvl128b -mabi=lp64d")
+  endif()
+endif()
+if (${CORE} STREQUAL RISCV64_GENERIC)
+  set (CCOMMON_OPT "${CCOMMON_OPT} -march=rv64imafdc -mabi=lp64d")
+endif()
+if (${CORE} STREQUAL x280)
+  set (CCOMMON_OPT "${CCOMMON_OPT} -march=rv64imafdcv_zba_zbb_zfh_zvl512b -mabi=lp64d")
+endif()
+
+
 if (NOT DYNAMIC_ARCH)
 	if (HAVE_AVX2)
         set (CCOMMON_OPT  "${CCOMMON_OPT} -mavx2")

cmake/kernel.cmake

@@ -175,6 +175,10 @@ if (BUILD_BFLOAT16)
   SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
   SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
 endif ()
+if (BUILD_HFLOAT16)
+  SetFallback(SHGEMVNKERNEL ../generic/gemv_n.c)
+  SetFallback(SHGEMVTKERNEL ../generic/gemv_t.c)
+endif ()
 endmacro ()
 
 macro(SetDefaultL2)
@@ -228,6 +232,10 @@ if (BUILD_BFLOAT16)
   SetFallback(BGEMVTKERNEL ../generic/gemv_t.c)
   SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
   SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
+endif ()
+if (BUILD_HFLOAT16)
+  SetFallback(SHGEMVNKERNEL ../generic/gemv_n.c)
+  SetFallback(SHGEMVTKERNEL ../generic/gemv_t.c)
   SetFallback(SHGERKERNEL ../generic/ger.c)
 endif ()
 endmacro ()
@@ -260,5 +268,16 @@ if (BUILD_BFLOAT16)
   SetFallback(SBGEMMONCOPYOBJ sbgemm_oncopy.o)
   SetFallback(SBGEMMOTCOPYOBJ sbgemm_otcopy.o)
 endif ()
-
+if (BUILD_HFLOAT16)
+  SetFallback(SHGEMMKERNEL ../generic/gemmkernel_2x2.c)
+  SetFallback(SHGEMM_BETA  ../generic/gemm_beta.c)
+  SetFallback(SHGEMMINCOPY ../generic/gemm_ncopy_2.c)
+  SetFallback(SHGEMMITCOPY ../generic/gemm_tcopy_2.c)
+  SetFallback(SHGEMMONCOPY ../generic/gemm_ncopy_2.c)
+  SetFallback(SHGEMMOTCOPY ../generic/gemm_tcopy_2.c)
+  SetFallback(SHGEMMINCOPYOBJ shgemm_incopy.o)
+  SetFallback(SHGEMMITCOPYOBJ shgemm_itcopy.o)
+  SetFallback(SHGEMMONCOPYOBJ shgemm_oncopy.o)
+  SetFallback(SHGEMMOTCOPYOBJ shgemm_otcopy.o)
+endif ()
 endmacro ()

cmake/prebuild.cmake

@@ -99,7 +99,6 @@ if (${COMPILER_ID} STREQUAL "GNU")
 endif ()
 
 string(TOUPPER ${ARCH} UC_ARCH)
-
 file(WRITE ${TARGET_CONF_TEMP}
   "#define OS_${HOST_OS}\t1\n"
   "#define ARCH_${UC_ARCH}\t1\n"
@@ -111,6 +110,10 @@ if (${HOST_OS} STREQUAL "WINDOWSSTORE")
   file(APPEND ${TARGET_CONF_TEMP}
     "#define OS_WINNT\t1\n")
 endif ()
+if (${HOST_OS} STREQUAL CYGWIN)
+  file(APPEND ${TARGET_CONF_TEMP}
+    "#define OS_CYGWIN_NT\t1\n")
+endif ()
 
 # f_check
 if (NOT NOFORTRAN)

cmake/system.cmake

@@ -381,6 +381,28 @@ if (${TARGET} STREQUAL NEOVERSEV1)
     endif()
   endif()
 
+  if ((${TARGET} STREQUAL RISCV64_ZVL128B) OR (${TARGET} STREQUAL RISCV64_ZVL256B))
+    set (RISCV64_OPT "rv64imafdcv")
+    if (BUILD_BFLOAT16)
+      set (RISCV64_OPT "${RISCV64_OPT}_zvfbfwma")
+    endif()
+    if (BUILD_HFLOAT16)
+      set (RISCV64_OPT "${RISCV64_OPT}_zvfh_zfh")
+    endif()
+    if (${TARGET} STREQUAL RISCV64_ZVL256B)
+      set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=${RISCV64_OPT}_zvl256b -mabi=lp64d")
+    endif()
+    if (${TARGET} STREQUAL RISCV64_ZVL128B)
+      set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=${RISCV64_OPT}_zvl128b -mabi=lp64d")
+    endif()
+  endif()
+  if (${TARGET} STREQUAL RISCV64_GENERIC)
+      set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=rv64imafdc -mabi=lp64d")
+    endif()
+    if (${TARGET} STREQUAL x280)
+      set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=rv64imafdcv_zba_zbb_zfh_zvl512b -mabi=lp64d")
+    endif()
+
 endif()
 
 if (DEFINED BINARY)
@@ -425,10 +447,16 @@ if (USE_OPENMP)
   if (NOT NOFORTRAN)
     find_package(OpenMP COMPONENTS Fortran REQUIRED)
     # Avoid mixed OpenMP linkage
-    get_target_property(OMP_C_LIB OpenMP::OpenMP_C INTERFACE_LINK_LIBRARIES)
-    get_target_property(OMP_Fortran_LIB OpenMP::OpenMP_Fortran INTERFACE_LINK_LIBRARIES)
-    if (NOT OMP_C_LIB STREQUAL OMP_Fortran_LIB)
-      message(FATAL_ERROR "Multiple OpenMP runtime libraries detected. Mixed OpenMP runtime linkage is dangerous. You may pass -DOpenMP_LANG_LIB_NAMES and -DOpenMP_omp_LIBRARY to manually choose the OpenMP library.")
+    get_target_property(OMP_C_LIBS OpenMP::OpenMP_C INTERFACE_LINK_LIBRARIES)
+    get_target_property(OMP_F_LIBS OpenMP::OpenMP_Fortran INTERFACE_LINK_LIBRARIES)
+    if (NOT OMP_C_LIBS STREQUAL OMP_F_LIBS)
+      message(NOTICE
+        "CMake detected different OpenMP libraries for C and Fortran:\n"
+        "C=${OMP_C_LIBS}\n"
+        "Fortran=${OMP_F_LIBS}\n"
+        "In case you encounter issues, please check that this is correct.\n"
+        "You may pass -DOpenMP_<lang>_LIB_NAMES and -DOpenMP_<libname>_LIBRARY to cmake to manually force the OpenMP library."
+      )
     endif()
   endif ()
 endif ()

cmake/utils.cmake

@@ -375,9 +375,12 @@ function(GenerateNamedObjects sources_in)
       if (NOT no_float_type)
         string(SUBSTRING ${float_type} 0 1 float_char)
         string(TOLOWER ${float_char} float_char)
-	if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEM")
-		set (float_char "sb")
-	endif ()
+        if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEM")
+          set (float_char "sb")
+        endif ()
+        if (${float_type} STREQUAL "HFLOAT16" AND NOT "${defines_in}" MATCHES "HGEM")
+          set (float_char "sh")
+        endif ()
       endif ()
 
       if (NOT name_in)
@@ -427,6 +430,9 @@ function(GenerateNamedObjects sources_in)
       endif ()
       if (${float_type} STREQUAL "BFLOAT16")
         list(APPEND obj_defines "BFLOAT16")
+      endif ()
+	  if (${float_type} STREQUAL "HFLOAT16")
+        list(APPEND obj_defines "HFLOAT16")
       endif ()
       if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
         list(APPEND obj_defines "COMPLEX")

common.h

@@ -781,7 +781,7 @@ static __inline int readenv_atoi(char *env) {
 #endif
 #endif
 
-#if !defined(XDOUBLE) || !defined(QUAD_PRECISION)
+#if !defined(BFLOAT16) && (!defined(XDOUBLE) || !defined(QUAD_PRECISION))
 
 static __inline void compinv(FLOAT *b, FLOAT ar, FLOAT ai){
 

common_arm64.h

@@ -114,9 +114,9 @@ static __inline BLASULONG rpcc(void){
   #else
   BLASULONG ret = 0;
   blasint shift;
- 
-  __asm__ __volatile__ ("isb; mrs %0,cntvct_el0":"=r"(ret));
-  __asm__ __volatile__ ("mrs %0,cntfrq_el0; clz %w0, %w0":"=&r"(shift));
+
+  __asm__ __volatile__ ("isb\n\tmrs %0,cntvct_el0":"=r"(ret));
+  __asm__ __volatile__ ("mrs %x0,cntfrq_el0\n\tclz %w0, %w0":"=&r"(shift));
 
   return ret << shift;
   #endif

common_interface.h

@@ -261,6 +261,8 @@ void BLASFUNC(bgemv)(char *, blasint *, blasint *, bfloat16  *, bfloat16 *, blas
             bfloat16  *, blasint *, bfloat16  *, bfloat16  *, blasint *);
 void BLASFUNC(sbgemv)(char *, blasint *, blasint *, float  *, bfloat16 *, blasint *,
             bfloat16  *, blasint *, float  *, float  *, blasint *);
+void BLASFUNC(shgemv)(char *, blasint *, blasint *, float  *, hfloat16 *, blasint *,
+            hfloat16  *, blasint *, float  *, float  *, blasint *);
 void BLASFUNC(sgemv)(char *, blasint *, blasint *, float  *, float  *, blasint *,
 		    float  *, blasint *, float  *, float  *, blasint *);
 void BLASFUNC(dgemv)(char *, blasint *, blasint *, double *, double *, blasint *,

common_level2.h

@@ -54,6 +54,10 @@ int sbgemv_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLO
 int sbgemv_t(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG);
 int sbgemv_thread_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG, int);
 int sbgemv_thread_t(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG, int);
+int shgemv_n(BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG);
+int shgemv_t(BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG);
+int shgemv_thread_n(BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG, int);
+int shgemv_thread_t(BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG, int);
 int sger_k (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *);
 int dger_k (BLASLONG, BLASLONG, BLASLONG, double, double *, BLASLONG, double *, BLASLONG, double *, BLASLONG, double *);
 int qger_k (BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble *);

common_level3.h

@@ -59,6 +59,57 @@ void sgemm_direct_alpha_beta(BLASLONG M, BLASLONG N, BLASLONG K,
 	float beta,
 	float * R, BLASLONG strideR);
 
+void ssymm_direct_alpha_betaLU(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB,
+	float beta,
+	float * R, BLASLONG strideR);
+void ssymm_direct_alpha_betaLL(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB,
+	float beta,
+	float * R, BLASLONG strideR);
+
+void strmm_direct_LNUN(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB);
+void strmm_direct_LNLN(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB);
+void strmm_direct_LTUN(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB);
+void strmm_direct_LTLN(BLASLONG M, BLASLONG N,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB);
+
+void ssyrk_direct_alpha_betaUN(BLASLONG N, BLASLONG K,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float beta,
+	float * C, BLASLONG strideC);
+void ssyrk_direct_alpha_betaUT(BLASLONG N, BLASLONG K,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float beta,
+	float * C, BLASLONG strideC);
+void ssyrk_direct_alpha_betaLN(BLASLONG N, BLASLONG K,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float beta,
+	float * C, BLASLONG strideC);
+void ssyrk_direct_alpha_betaLT(BLASLONG N, BLASLONG K,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float beta,
+	float * C, BLASLONG strideC);
+
 int sgemm_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K);
 
 int shgemm_beta(BLASLONG, BLASLONG, BLASLONG, float,

common_macro.h

@@ -703,6 +703,9 @@
 #define GEMM_THREAD_RC   SHGEMM_THREAD_NT
 #define GEMM_THREAD_RR   SHGEMM_THREAD_NN
 
+#define SCAL_K        SSCAL_K
+#define GEMV_N        SHGEMV_N_K
+#define GEMV_T        SHGEMV_T_K
 
 #elif defined(BFLOAT16) && defined(BGEMM)
 #define SCAL_K        BSCAL_K

common_param.h

@@ -60,7 +60,8 @@ int (*shgemm_itcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 int (*shgemm_oncopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 
-
+int (*shgemv_n) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG);
+int (*shgemv_t) (BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float, float *, BLASLONG);
 #endif
 
 
@@ -257,6 +258,16 @@ int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 #ifdef ARCH_ARM64
   void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG);
   void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG);
+  void (*ssymm_direct_alpha_betaLU) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG);
+  void (*ssymm_direct_alpha_betaLL) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG);
+  void (*strmm_direct_LNUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG);
+  void (*strmm_direct_LNLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG);
+  void (*strmm_direct_LTUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG);
+  void (*strmm_direct_LTLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG);
+  void (*ssyrk_direct_alpha_betaUN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG);
+  void (*ssyrk_direct_alpha_betaUT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG);
+  void (*ssyrk_direct_alpha_betaLN) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG);
+  void (*ssyrk_direct_alpha_betaLT) (BLASLONG, BLASLONG, float, float *, BLASLONG, float, float *, BLASLONG);
 #endif
 
   

common_s.h

@@ -50,6 +50,16 @@
 #define SGEMM_DIRECT_PERFORMANT    sgemm_direct_performant
 #define SGEMM_DIRECT		sgemm_direct
 #define  SGEMM_DIRECT_ALPHA_BETA		sgemm_direct_alpha_beta
+#define  SSYMM_DIRECT_ALPHA_BETA_LU		ssymm_direct_alpha_betaLU
+#define  SSYMM_DIRECT_ALPHA_BETA_LL		ssymm_direct_alpha_betaLL
+#define  STRMM_DIRECT_LNUN		strmm_direct_LNUN
+#define  STRMM_DIRECT_LNLN		strmm_direct_LNLN
+#define  STRMM_DIRECT_LTUN		strmm_direct_LTUN
+#define  STRMM_DIRECT_LTLN		strmm_direct_LTLN
+#define  SSYRK_DIRECT_ALPHA_BETA_UN         ssyrk_direct_alpha_betaUN
+#define  SSYRK_DIRECT_ALPHA_BETA_UT         ssyrk_direct_alpha_betaUT
+#define  SSYRK_DIRECT_ALPHA_BETA_LN         ssyrk_direct_alpha_betaLN
+#define  SSYRK_DIRECT_ALPHA_BETA_LT         ssyrk_direct_alpha_betaLT
 
 #define	SGEMM_ONCOPY		sgemm_oncopy
 #define	SGEMM_OTCOPY		sgemm_otcopy
@@ -220,6 +230,16 @@
 #define SGEMM_DIRECT_PERFORMANT    sgemm_direct_performant
 #define  SGEMM_DIRECT		gotoblas -> sgemm_direct
 #define  SGEMM_DIRECT_ALPHA_BETA		gotoblas -> sgemm_direct_alpha_beta
+#define  SSYMM_DIRECT_ALPHA_BETA_LU		gotoblas -> ssymm_direct_alpha_betaLU
+#define  SSYMM_DIRECT_ALPHA_BETA_LL		gotoblas -> ssymm_direct_alpha_betaLL
+#define  STRMM_DIRECT_LNUN		gotoblas -> strmm_direct_LNUN
+#define  STRMM_DIRECT_LNLN		gotoblas -> strmm_direct_LNLN
+#define  STRMM_DIRECT_LTUN		gotoblas -> strmm_direct_LTUN
+#define  STRMM_DIRECT_LTLN		gotoblas -> strmm_direct_LTLN
+#define  SSYRK_DIRECT_ALPHA_BETA_UN     gotoblas -> ssyrk_direct_alpha_betaUN
+#define  SSYRK_DIRECT_ALPHA_BETA_UT     gotoblas -> ssyrk_direct_alpha_betaUT
+#define  SSYRK_DIRECT_ALPHA_BETA_LN     gotoblas -> ssyrk_direct_alpha_betaLN
+#define  SSYRK_DIRECT_ALPHA_BETA_LT     gotoblas -> ssyrk_direct_alpha_betaLT
 #endif
 
 #define	SGEMM_ONCOPY		gotoblas -> sgemm_oncopy

common_sh.h

@@ -1,3 +1,31 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
 #ifndef COMMON_SH_H
 #define COMMON_SH_H
 
@@ -17,6 +45,9 @@
 #define SHGEMM_BETA		    shgemm_beta
 #define SHGEMM_KERNEL       shgemm_kernel
 
+#define SHGEMV_N_K      shgemv_n
+#define SHGEMV_T_K      shgemv_t
+
 
 #else // #DYNAMIC_ARCH
 
@@ -32,6 +63,10 @@
 
 #define SHGEMM_BETA		gotoblas -> shgemm_beta
 #define SHGEMM_KERNEL		gotoblas -> shgemm_kernel
+
+#define SHGEMV_N_K      gotoblas->shgemv_n
+#define SHGEMV_T_K      gotoblas->shgemv_t
+
 #endif // #DYNAMIC_ARCH
 
 #define SHGEMM_NN        shgemm_nn

cpuid_arm64.c

@@ -150,6 +150,8 @@ static int cpulowperf=0;
 static int cpumidperf=0;
 static int cpuhiperf=0;
 
+int aliased = 0;
+
 int get_feature(char *search)
 {
 
@@ -346,7 +348,7 @@ int detect(void)
     else if (strstr(cpu_implementer, "0x46") && strstr(cpu_part, "0x001"))
                         return CPU_A64FX;
     // Apple
-    else if (strstr(cpu_implementer, "0x61") && strstr(cpu_part, "0x022"))
+    else if (strstr(cpu_implementer, "0x61") /* && strstr(cpu_part, "0x022")*/)
 	    		return CPU_VORTEX;
    // Phytium
    else if (strstr(cpu_implementer, "0x70") && (strstr(cpu_part, "0x660") || strstr(cpu_part, "0x661") 
@@ -418,7 +420,10 @@ int detect(void)
 	if (errcode != ERROR_SUCCESS) wprintf(L"Error reading cpuname from registry:%x\n",errcode);
 //wprintf(stderr,L"%s\n",(PWSTR)valstring);
 	RegCloseKey(reghandle);
-	if (strstr(valstring, "Snapdragon(R) X Elite")) return CPU_NEOVERSEN1;
+	if (strstr(valstring, "Snapdragon(R) X Elite")) {
+		aliased = 1;
+		return CPU_NEOVERSEN1;
+	}
 	if (strstr(valstring, "Ampere(R) Altra")) return CPU_NEOVERSEN1;
 	if (strstr(valstring, "Snapdragon (TM) 8cx Gen 3")) return CPU_CORTEXX1;
 	if (strstr(valstring, "Snapdragon Compute Platform")) return CPU_CORTEXX1;
@@ -541,6 +546,7 @@ void get_cpuconfig(void)
 		break;
 	    case CPU_NEOVERSEN1:
 		printf("#define %s\n", cpuname[d]);
+		if (aliased == 0) {
 		printf("#define L1_CODE_SIZE 65536\n");
 		printf("#define L1_CODE_LINESIZE 64\n");
 		printf("#define L1_CODE_ASSOCIATIVE 4\n");
@@ -552,6 +558,23 @@ void get_cpuconfig(void)
 		printf("#define L2_ASSOCIATIVE 8\n");
 		printf("#define DTB_DEFAULT_ENTRIES 48\n");
 		printf("#define DTB_SIZE 4096\n");
+		} else {
+		printf("#define L1_CODE_SIZE 196608\n");
+		printf("#define L1_CODE_LINESIZE 64\n");
+		printf("#define L1_CODE_ASSOCIATIVE 6\n");
+		printf("#define L1_DATA_SIZE 98304\n");
+		printf("#define L1_DATA_LINESIZE 64\n");
+		printf("#define L1_DATA_ASSOCIATIVE 6\n");
+		printf("#define L2_SIZE 12582912\n");
+		printf("#define L2_LINESIZE 32\n");
+		printf("#define L2_ASSOCIATIVE 12\n");
+		printf("#define ITB_SIZE 4096\n");
+		printf("#define ITB_ASSOCIATIVE 8\n");
+		printf("#define ITB_DEFAULT_ENTRIES 256\n");
+		printf("#define DTB_DEFAULT_ENTRIES 224\n");
+		printf("#define DTB_ASSOCIATIVE 7\n");
+		printf("#define DTB_SIZE 4096\n");
+		}	
 		break;
 
 	    case CPU_NEOVERSEV1:

driver/level2/CMakeLists.txt

@@ -202,6 +202,15 @@ if (BUILD_BFLOAT16)
   if (USE_THREAD)
     GenerateNamedObjects("sbgemv_thread.c" "" "gemv_thread_n" false "" "" false "BFLOAT16")
     GenerateNamedObjects("sbgemv_thread.c" "TRANSA" "gemv_thread_t" false "" "" false "BFLOAT16")
+    GenerateNamedObjects("sbgemv_thread.c" "BGEMM;BFLOAT16" "bgemv_thread_n" false "" "" true "")
+    GenerateNamedObjects("sbgemv_thread.c" "BGEMM;BFLOAT16;TRANSA" "bgemv_thread_t" false "" "" true "")
+  endif ()
+endif ()
+
+if (BUILD_HFLOAT16)
+  if (USE_THREAD)
+    GenerateNamedObjects("sbgemv_thread.c" "" "gemv_thread_n" false "" "" false "HFLOAT16")
+    GenerateNamedObjects("sbgemv_thread.c" "TRANSA" "gemv_thread_t" false "" "" false "HFLOAT16")
   endif ()
 endif ()
 

driver/level2/Makefile

@@ -450,6 +450,12 @@ XBLASOBJS   += \
 	xtbmv_thread_CUU.$(SUFFIX)	xtbmv_thread_CUN.$(SUFFIX) \
 	xtbmv_thread_CLU.$(SUFFIX)	xtbmv_thread_CLN.$(SUFFIX)
 
+
+ifeq ($(BUILD_HFLOAT16),1)
+SHBLASOBJS     += \
+        shgemv_thread_n$(TSUFFIX).$(SUFFIX) \
+        shgemv_thread_t$(TSUFFIX).$(SUFFIX)
+endif
 ifeq ($(BUILD_BFLOAT16),1)
 BBLASOBJS      += \
         bgemv_thread_n$(TSUFFIX).$(SUFFIX) \
@@ -3737,6 +3743,13 @@ xtrsv_CUU.$(SUFFIX)  xtrsv_CUU.$(PSUFFIX)  : ztrsv_L.c ../../param.h
 xtrsv_CUN.$(SUFFIX)  xtrsv_CUN.$(PSUFFIX)  : ztrsv_L.c ../../param.h
 	$(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DTRANSA=4 -UUNIT $< -o $(@F)
 
+ifeq ($(BUILD_HFLOAT16),1)
+shgemv_thread_n.$(SUFFIX) shgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h
+	$(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE  -UTRANSA -UCONJ -UXCONJ $< -o $(@F)
+shgemv_thread_t.$(SUFFIX) shgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h
+	$(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE  -DTRANSA -UCONJ -UXCONJ $< -o $(@F)
+endif
+
 ifeq ($(BUILD_BFLOAT16),1)
 bgemv_thread_n.$(SUFFIX) bgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h
 	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE  -UTRANSA -UCONJ -UXCONJ $< -o $(@F)

driver/level3/CMakeLists.txt

@@ -17,6 +17,7 @@ foreach (GEMM_DEFINE ${GEMM_DEFINES})
     GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};BGEMM" "gemm_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
     if (USE_THREAD AND NOT USE_SIMPLE_THREADED_LEVEL3)
       GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3" "gemm_thread_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
+      GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3;BGEMM" "gemm_thread_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
     endif ()
   endif ()
   if (BUILD_HFLOAT16)

driver/level3/level3_thread.c

@@ -588,8 +588,6 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
   InitializeCriticalSection((PCRITICAL_SECTION)&level3_lock);
 #else
   static pthread_mutex_t  level3_lock    = PTHREAD_MUTEX_INITIALIZER;
-  static pthread_cond_t  level3_wakeup    = PTHREAD_COND_INITIALIZER;
-  volatile static BLASLONG CPU_AVAILABLE = MAX_CPU_NUMBER;
 #endif
 
   blas_arg_t newarg;
@@ -659,12 +657,6 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
   EnterCriticalSection((PCRITICAL_SECTION)&level3_lock);
 #else
   pthread_mutex_lock(&level3_lock);
-  while(CPU_AVAILABLE < nthreads) { 
-    pthread_cond_wait(&level3_wakeup, &level3_lock);
-  } 
-  CPU_AVAILABLE -= nthreads;
-  WMB;
-  pthread_mutex_unlock(&level3_lock);
 #endif
 
 #ifdef USE_ALLOC_HEAP
@@ -816,10 +808,6 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
 #elif defined(OS_WINDOWS)
   LeaveCriticalSection((PCRITICAL_SECTION)&level3_lock);
 #else
-  pthread_mutex_lock(&level3_lock);
-  CPU_AVAILABLE += nthreads;
-  WMB;
-  pthread_cond_signal(&level3_wakeup);
   pthread_mutex_unlock(&level3_lock);
 #endif
 

driver/others/CMakeLists.txt

@@ -13,6 +13,8 @@ if (USE_THREAD)
     set(BLAS_SERVER blas_server_omp.c)
   elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
     set(BLAS_SERVER blas_server_win32.c)
+  elseif (${CMAKE_SYSTEM_NAME} STREQUAL CYGWIN)
+    set(BLAS_SERVER blas_server_win32.c)
   elseif (${CMAKE_SYSTEM_NAME} STREQUAL "WindowsStore")
     set(BLAS_SERVER blas_server_win32.c)
   endif ()

driver/others/blas_server.c

@@ -637,9 +637,7 @@ int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
 
 #ifdef SMP_SERVER
   // Handle lazy re-init of the thread-pool after a POSIX fork
-  LOCK_COMMAND(&server_lock);
   if (unlikely(blas_server_avail == 0)) blas_thread_init();
-  UNLOCK_COMMAND(&server_lock);
 #endif
   BLASLONG i = 0;
   blas_queue_t *current = queue;

driver/others/dynamic_riscv64.c

@@ -99,6 +99,7 @@ struct riscv_hwprobe {
 #define		RISCV_HWPROBE_IMA_V		(1 << 2)
 #define		RISCV_HWPROBE_EXT_ZFH		(1 << 27)
 #define		RISCV_HWPROBE_EXT_ZVFH		(1 << 30)
+#define		RISCV_HWPROBE_EXT_ZVFBFWMA	(1 << 54)
 
 #ifndef NR_riscv_hwprobe
 #ifndef NR_arch_specific_syscall
@@ -170,6 +171,8 @@ static gotoblas_t* get_coretype(void) {
 	if (ret == 0) {
 #if defined(BUILD_HFLOAT16)
 		vector_mask = (RISCV_HWPROBE_IMA_V | RISCV_HWPROBE_EXT_ZFH | RISCV_HWPROBE_EXT_ZVFH);
+#elif defined(BUILD_BFLOAT16)
+		vector_mask = (RISCV_HWPROBE_IMA_V | RISCV_HWPROBE_EXT_ZVFBFWMA);
 #else
 		vector_mask = RISCV_HWPROBE_IMA_V;
 #endif
@@ -180,6 +183,10 @@ static gotoblas_t* get_coretype(void) {
 		snprintf(coremsg, sizeof(coremsg), "Cpu support for Zfh+Zvfh extensions required due to BUILD_HFLOAT16=1\n");
 		openblas_warning(1, coremsg);
 		return NULL;
+#elif defined(BUILD_BFLOAT16)
+		snprintf(coremsg, sizeof(coremsg), "Cpu support for Zvfbfwma extensions required due to BUILD_BFLOAT16=1\n");
+		openblas_warning(1, coremsg);
+		return NULL;
 #else
 		if (!(getauxval(AT_HWCAP) & DETECT_RISCV64_HWCAP_ISA_V))
 			return NULL;

exports/gensymbol

@@ -80,7 +80,7 @@ blasobjsz="
 
 blasobjs="lsame xerbla"
 bfblasobjs="bgemm bgemv sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod"
-hfblasobjs="shgemm"
+hfblasobjs="shgemm shgemv"
 cblasobjsc="
     cblas_caxpy cblas_ccopy cblas_cdotc cblas_cdotu cblas_cgbmv cblas_cgemm cblas_cgemv
     cblas_cgerc cblas_cgeru cblas_chbmv cblas_chemm cblas_chemv cblas_cher2 cblas_cher2k

exports/gensymbol.pl

@@ -80,7 +80,7 @@
 
 @blasobjs = (lsame, xerbla);
 @bfblasobjs = (bgemm, bgemv, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod);
-@hfblasobjs = (shgemm);
+@hfblasobjs = (shgemm, shgemv);
 @cblasobjsc = (
     cblas_caxpy, cblas_ccopy, cblas_cdotc, cblas_cdotu, cblas_cgbmv, cblas_cgemm, cblas_cgemv,
     cblas_cgerc, cblas_cgeru, cblas_chbmv, cblas_chemm, cblas_chemv, cblas_cher2, cblas_cher2k,

f_check

@@ -30,9 +30,11 @@ nofortran=0
 shift 2
 compiler="$*"
 compiler_bin="$1"
+shift
+compiler_args="$*"
 
 # f77 is too ambiguous
-[ "$compiler" = "f77" ] && compiler=''
+[ "$compiler_bin" = "f77" ] && compiler=''
 
 path=`split "$PATH" ':'`
 
@@ -50,7 +52,7 @@ if [ -z "$compiler" ]; then
     for list in $lists; do
         for p in $path; do
             if [ -x "$p/$list" ]; then
-                compiler=$list
+                compiler="$list $compiler_args"
                 compiler_bin=$list
                 break 2
             fi

getarch.c

@@ -2046,10 +2046,9 @@ int main(int argc, char *argv[]){
 #endif
 
 
-#ifdef INTEL_AMD
-#ifndef FORCE
+#if defined(INTEL_AMD) && !defined(FORCE)
     get_sse();
-#else
+#elif defined(FORCE_INTEL)
 
     sprintf(buffer, "%s", ARCHCONFIG);
 
@@ -2079,7 +2078,6 @@ int main(int argc, char *argv[]){
       } else p ++;
     }
 #endif
-#endif
 
 #if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
 printf("__BYTE_ORDER__=__ORDER_BIG_ENDIAN__\n");

interface/CMakeLists.txt

@@ -125,6 +125,7 @@ foreach (CBLAS_FLAG ${CBLAS_FLAGS})
   if (BUILD_SINGLE OR BUILD_DOUBLE)
   GenerateNamedObjects("sdsdot.c" "" "sdsdot" ${CBLAS_FLAG} "" "" true "SINGLE")
   GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" ${CBLAS_FLAG} "" "" false)
+  GenerateNamedObjects("gemm_batch_strided.c" "" "gemm_batch_strided" ${CBLAS_FLAG} "" "" false)
   endif ()
 if (BUILD_DOUBLE)
   GenerateNamedObjects("dsdot.c" "" "dsdot" ${CBLAS_FLAG} "" "" true "SINGLE")
@@ -161,9 +162,11 @@ if (BUILD_BFLOAT16)
 	GenerateNamedObjects("bf16to.c" "SINGLE_PREC" "sbf16tos" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("bf16to.c" "DOUBLE_PREC" "dbf16tod" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("gemm_batch.c" "" "sbgemm_batch" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("gemm_batch_strided.c" "" "sbgemm_batch_strided" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 endif ()
 if (BUILD_HFLOAT16)
   GenerateNamedObjects("gemm.c" "" "shgemm" ${CBLAS_FLAG} "" "" true "HFLOAT16")
+  GenerateNamedObjects("sbgemv.c" "" "shgemv" ${CBLAS_FLAG} "" "" true "HFLOAT16")
 endif ()
 
 # complex-specific sources
@@ -194,6 +197,7 @@ foreach (float_type ${FLOAT_TYPES})
     GenerateNamedObjects("asum.c" "" "scasum" ${CBLAS_FLAG} "" "" true "COMPLEX")
     GenerateNamedObjects("sum.c" "" "scsum" ${CBLAS_FLAG} "" "" true "COMPLEX")
     GenerateNamedObjects("gemm_batch.c" "" "cgemm_batch" ${CBLAS_FLAG} "" "" true "COMPLEX")
+    GenerateNamedObjects("gemm_batch_strided.c" "" "cgemm_batch_strided" ${CBLAS_FLAG} "" "" true "COMPLEX")
   endif ()
   if (${float_type} STREQUAL "ZCOMPLEX")
     GenerateNamedObjects("zscal.c" "SSCAL" "dscal" ${CBLAS_FLAG} "" "" false "ZCOMPLEX")
@@ -204,6 +208,7 @@ foreach (float_type ${FLOAT_TYPES})
     GenerateNamedObjects("asum.c" "" "dzasum" ${CBLAS_FLAG} "" "" true "ZCOMPLEX")
     GenerateNamedObjects("sum.c" "" "dzsum" ${CBLAS_FLAG} "" "" true "ZCOMPLEX")
     GenerateNamedObjects("gemm_batch.c" "" "zgemm_batch" ${CBLAS_FLAG} "" "" true "ZCOMPLEX")
+    GenerateNamedObjects("gemm_batch_strided.c" "" "zgemm_batch_strided" ${CBLAS_FLAG} "" "" true "ZCOMPLEX")
   endif ()
 endforeach ()
 
@@ -255,6 +260,7 @@ if ( BUILD_COMPLEX AND NOT  BUILD_SINGLE)
 	GenerateNamedObjects("gemv.c" "" "gemv" 0 "" "" false "SINGLE")
 	GenerateNamedObjects("gemm.c" "" "gemm" 0 "" "" false "SINGLE")
 	GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" 0 "" "" false "SINGLE")
+	GenerateNamedObjects("gemm_batch_strided.c" "" "gemm_batch_strided" 0 "" "" false "SINGLE")
 	GenerateNamedObjects("asum.c" "" "asum" 0 "" "" false "SINGLE")
 	GenerateNamedObjects("swap.c" "" "swap" 0 "" "" false "SINGLE")
 	GenerateNamedObjects("axpy.c" "" "axpy" 0 "" "" false "SINGLE")
@@ -269,6 +275,7 @@ if ( BUILD_COMPLEX16 AND NOT  BUILD_DOUBLE)
 	GenerateNamedObjects("gemv.c" "" "gemv" 0 "" "" false "DOUBLE")
 	GenerateNamedObjects("gemm.c" "" "gemm" 0 "" "" false "DOUBLE")
 	GenerateNamedObjects("gemm_batch.c" "" "gemm_batch" 0 "" "" false "DOUBLE")
+	GenerateNamedObjects("gemm_batch_strided.c" "" "gemm_batch_strided" 0 "" "" false "DOUBLE")
 	GenerateNamedObjects("asum.c" "" "asum" 0 "" "" false "DOUBLE")
 	GenerateNamedObjects("swap.c" "" "swap" 0 "" "" false "DOUBLE")
 	GenerateNamedObjects("axpy.c" "" "axpy" 0 "" "" false "DOUBLE")

interface/Makefile

@@ -73,7 +73,7 @@ SBLAS3OBJS    = \
 		strsm.$(SUFFIX) ssyrk.$(SUFFIX) ssyr2k.$(SUFFIX) \
 		somatcopy.$(SUFFIX) simatcopy.$(SUFFIX)\
 		sgeadd.$(SUFFIX) sgemmt.$(SUFFIX) sgemmtr.$(SUFFIX) \
-		sgemm_batch.$(SUFFIX)
+		sgemm_batch.$(SUFFIX) sgemm_batch_strided.$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
 BBLAS3OBJS     = bgemm.$(SUFFIX)
@@ -81,12 +81,13 @@ BBLAS2OBJS     = bgemv.$(SUFFIX)
 BBLAS1OBJS     = bscal.$(SUFFIX)
 SBBLAS1OBJS    = sbdot.$(SUFFIX)
 SBBLAS2OBJS    = sbgemv.$(SUFFIX)
-SBBLAS3OBJS    = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX) sbgemm_batch.$(SUFFIX)
+SBBLAS3OBJS    = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX) sbgemm_batch.$(SUFFIX) sbgemm_batch_strided.$(SUFFIX)
 SBEXTOBJS      = sbstobf16.$(SUFFIX) sbdtobf16.$(SUFFIX) sbf16tos.$(SUFFIX) dbf16tod.$(SUFFIX)
 endif
 
 ifeq ($(BUILD_HFLOAT16),1)
 SHBLAS3OBJS    = shgemm.$(SUFFIX)
+SHBLAS2OBJS    = shgemv.$(SUFFIX)
 endif
 
 DBLAS1OBJS    = \
@@ -113,7 +114,7 @@ DBLAS3OBJS    = \
 		dtrsm.$(SUFFIX) dsyrk.$(SUFFIX) dsyr2k.$(SUFFIX) \
 		domatcopy.$(SUFFIX) dimatcopy.$(SUFFIX)\
 		dgeadd.$(SUFFIX) dgemmt.$(SUFFIX) dgemmtr.$(SUFFIX) \
-		dgemm_batch.$(SUFFIX)
+		dgemm_batch.$(SUFFIX) dgemm_batch_strided.$(SUFFIX)
 
 CBLAS1OBJS    = \
 		caxpy.$(SUFFIX) caxpyc.$(SUFFIX) cswap.$(SUFFIX) \
@@ -143,7 +144,7 @@ CBLAS3OBJS    = \
 	       	chemm.$(SUFFIX) cherk.$(SUFFIX) cher2k.$(SUFFIX) \
 		comatcopy.$(SUFFIX) cimatcopy.$(SUFFIX)\
 		cgeadd.$(SUFFIX) cgemmt.$(SUFFIX) cgemmtr.$(SUFFIX) \
-		cgemm_batch.$(SUFFIX)
+		cgemm_batch.$(SUFFIX) cgemm_batch_strided.$(SUFFIX)
 
 ZBLAS1OBJS    = \
 		zaxpy.$(SUFFIX) zaxpyc.$(SUFFIX) zswap.$(SUFFIX) \
@@ -173,7 +174,7 @@ ZBLAS3OBJS    = \
 	       	zhemm.$(SUFFIX) zherk.$(SUFFIX) zher2k.$(SUFFIX) \
 		zomatcopy.$(SUFFIX) zimatcopy.$(SUFFIX)\
 		zgeadd.$(SUFFIX) zgemmt.$(SUFFIX) zgemmtr.$(SUFFIX) \
-		zgemm_batch.$(SUFFIX)
+		zgemm_batch.$(SUFFIX) zgemm_batch_strided.$(SUFFIX)
 
 ifeq ($(SUPPORT_GEMM3M), 1)
 
@@ -321,7 +322,7 @@ CSBLAS2OBJS   = \
 CSBLAS3OBJS   = \
 	cblas_sgemm.$(SUFFIX) cblas_ssymm.$(SUFFIX) cblas_strmm.$(SUFFIX) cblas_strsm.$(SUFFIX) \
 	cblas_ssyrk.$(SUFFIX) cblas_ssyr2k.$(SUFFIX) cblas_somatcopy.$(SUFFIX)  cblas_simatcopy.$(SUFFIX)\
-	cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemmtr.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX)
+	cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemmtr.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX) cblas_sgemm_batch_strided.$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
 CBBLAS3OBJS = cblas_bgemm.$(SUFFIX)
@@ -329,7 +330,7 @@ CBBLAS2OBJS = cblas_bgemv.$(SUFFIX)
 CBBLAS1OBJS = cblas_bscal.$(SUFFIX)
 CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX)
 CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX)
-CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX)
+CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX) cblas_sbgemm_batch_strided.$(SUFFIX)
 CSBEXTOBJS   = cblas_sbstobf16.$(SUFFIX) cblas_sbdtobf16.$(SUFFIX) cblas_sbf16tos.$(SUFFIX) cblas_dbf16tod.$(SUFFIX)
 ifeq ($(ONLY_CBLAS),1)
 CSBEXTOBJS    += sbstobf16.$(SUFFIX) sbdtobf16.$(SUFFIX) sbf16tos.$(SUFFIX) dbf16tod.$(SUFFIX)
@@ -338,6 +339,7 @@ endif
 
 ifeq ($(BUILD_HFLOAT16),1)
 CSHBLAS3OBJS = cblas_shgemm.$(SUFFIX)
+CSHBLAS2OBJS = cblas_shgemv.$(SUFFIX)
 endif
 
 CDBLAS1OBJS   = \
@@ -357,7 +359,7 @@ CDBLAS2OBJS   = \
 CDBLAS3OBJS   += \
 	cblas_dgemm.$(SUFFIX) cblas_dsymm.$(SUFFIX) cblas_dtrmm.$(SUFFIX) cblas_dtrsm.$(SUFFIX) \
 	cblas_dsyrk.$(SUFFIX) cblas_dsyr2k.$(SUFFIX) cblas_domatcopy.$(SUFFIX)  cblas_dimatcopy.$(SUFFIX) \
-        cblas_dgeadd.$(SUFFIX) cblas_dgemmt.$(SUFFIX) cblas_dgemmtr.$(SUFFIX) cblas_dgemm_batch.$(SUFFIX)
+        cblas_dgeadd.$(SUFFIX) cblas_dgemmt.$(SUFFIX) cblas_dgemmtr.$(SUFFIX) cblas_dgemm_batch.$(SUFFIX) cblas_dgemm_batch_strided.$(SUFFIX)
 
 CCBLAS1OBJS   = \
 	cblas_icamax.$(SUFFIX) cblas_icamin.$(SUFFIX) cblas_scasum.$(SUFFIX)  cblas_caxpy.$(SUFFIX) \
@@ -382,7 +384,7 @@ CCBLAS3OBJS   = \
 	cblas_csyrk.$(SUFFIX) cblas_csyr2k.$(SUFFIX) \
 	cblas_chemm.$(SUFFIX) cblas_cherk.$(SUFFIX) cblas_cher2k.$(SUFFIX) \
 	cblas_comatcopy.$(SUFFIX) cblas_cimatcopy.$(SUFFIX)\
-	cblas_cgeadd.$(SUFFIX) cblas_cgemmt.$(SUFFIX) cblas_cgemmtr.$(SUFFIX) cblas_cgemm_batch.$(SUFFIX)
+	cblas_cgeadd.$(SUFFIX) cblas_cgemmt.$(SUFFIX) cblas_cgemmtr.$(SUFFIX) cblas_cgemm_batch.$(SUFFIX) cblas_cgemm_batch_strided.$(SUFFIX)
 	
 CXERBLAOBJ = \
 	cblas_xerbla.$(SUFFIX)
@@ -413,7 +415,7 @@ CZBLAS3OBJS   = \
 	cblas_zsyrk.$(SUFFIX) cblas_zsyr2k.$(SUFFIX) \
 	cblas_zhemm.$(SUFFIX) cblas_zherk.$(SUFFIX) cblas_zher2k.$(SUFFIX)\
 	cblas_zomatcopy.$(SUFFIX) cblas_zimatcopy.$(SUFFIX) \
-	cblas_zgeadd.$(SUFFIX) cblas_zgemmt.$(SUFFIX) cblas_zgemmtr.$(SUFFIX) cblas_zgemm_batch.$(SUFFIX)
+	cblas_zgeadd.$(SUFFIX) cblas_zgemmt.$(SUFFIX) cblas_zgemmtr.$(SUFFIX) cblas_zgemm_batch.$(SUFFIX) cblas_zgemm_batch_strided.$(SUFFIX)
 
 
 ifeq ($(SUPPORT_GEMM3M), 1)
@@ -441,6 +443,7 @@ SBBLAS1OBJS  += $(CSBBLAS1OBJS)
 SBBLAS2OBJS  += $(CSBBLAS2OBJS)
 SBBLAS3OBJS  += $(CSBBLAS3OBJS)
 SHBLAS3OBJS  += $(CSHBLAS3OBJS)
+SHBLAS2OBJS  += $(CSHBLAS2OBJS)
 DBLAS1OBJS   += $(CDBLAS1OBJS)
 DBLAS2OBJS   += $(CDBLAS2OBJS)
 DBLAS3OBJS   += $(CDBLAS3OBJS)
@@ -459,7 +462,7 @@ endif
 BBLASOBJS    = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS)
 SBLASOBJS    = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS)
 SBBLASOBJS   = $(SBBLAS1OBJS) $(SBBLAS2OBJS) $(SBBLAS3OBJS)
-SHBLASOBJS   = $(SHBLAS3OBJS)
+SHBLASOBJS   = $(SHBLAS3OBJS) $(SHBLAS2OBJS)
 DBLASOBJS    = $(DBLAS1OBJS) $(DBLAS2OBJS) $(DBLAS3OBJS)
 QBLASOBJS    = $(QBLAS1OBJS) $(QBLAS2OBJS) $(QBLAS3OBJS)
 CBLASOBJS    = $(CBLAS1OBJS) $(CBLAS2OBJS) $(CBLAS3OBJS)
@@ -602,7 +605,7 @@ clean ::
 level1 : $(SBEXTOBJS) $(SBBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS)
 	$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
 
-level2 : $(SBBLAS2OBJS) $(BBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS)
+level2 : $(SBBLAS2OBJS) $(BBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) $(SHBLAS2OBJS)
 	$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
 
 level3 : $(SBBLAS3OBJS) $(BBLAS3OBJ) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS)
@@ -1002,6 +1005,11 @@ sbgemv.$(SUFFIX) sbgemv.$(PSUFFIX) : sbgemv.c
 	$(CC) $(CFLAGS) -c $< -o $(@F)
 endif
 
+ifeq ($(BUILD_HFLOAT16),1)
+shgemv.$(SUFFIX) shgemv.$(PSUFFIX) : sbgemv.c
+	$(CC) $(CFLAGS) -c $< -o $(@F)
+endif
+
 ifndef USE_NETLIB_GEMV
 sgemv.$(SUFFIX) sgemv.$(PSUFFIX): gemv.c
 	$(CC) -c $(CFLAGS) -o $(@F) $<
@@ -1832,6 +1840,11 @@ cblas_sbgemv.$(SUFFIX) cblas_sbgemv.$(PSUFFIX) : sbgemv.c
 	$(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F)
 endif
 
+ifeq ($(BUILD_HFLOAT16),1)
+cblas_shgemv.$(SUFFIX) cblas_shgemv.$(PSUFFIX) : sbgemv.c
+	$(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F)
+endif
+
 cblas_sgemv.$(SUFFIX) cblas_sgemv.$(PSUFFIX): gemv.c
 	$(CC) -DCBLAS -c $(CFLAGS) -o $(@F) $<
 
@@ -2544,6 +2557,21 @@ cblas_cgemm_batch.$(SUFFIX) cblas_cgemm_batch.$(PSUFFIX) : gemm_batch.c ../param
 cblas_zgemm_batch.$(SUFFIX) cblas_zgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h
 	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
 
+cblas_sbgemm_batch_strided.$(SUFFIX) cblas_sbgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
+
+cblas_sgemm_batch_strided.$(SUFFIX) cblas_sgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
+
+cblas_dgemm_batch_strided.$(SUFFIX) cblas_dgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
+
+cblas_cgemm_batch_strided.$(SUFFIX) cblas_cgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
+
+cblas_zgemm_batch_strided.$(SUFFIX) cblas_zgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -DCBLAS $< -o $(@F)
+
 sbgemm_batch.$(SUFFIX) sbgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h
 	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
 
@@ -2559,3 +2587,17 @@ cgemm_batch.$(SUFFIX) cgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h
 zgemm_batch.$(SUFFIX) zgemm_batch.$(PSUFFIX) : gemm_batch.c ../param.h
 	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
 
+sbgemm_batch_strided.$(SUFFIX) sbgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
+
+sgemm_batch_strided.$(SUFFIX) sgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
+
+dgemm_batch_strided.$(SUFFIX) dgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
+
+cgemm_batch_strided.$(SUFFIX) cgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)
+
+zgemm_batch_strided.$(SUFFIX) zgemm_batch_strided.$(PSUFFIX) : gemm_batch_strided.c ../param.h
+	$(CC) -c $(CFLAGS) -UCBLAS $< -o $(@F)

interface/gemm.c

@@ -587,7 +587,18 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
 	 args.m, args.n, args.k, args.lda, args.ldb, args.ldc);
 #endif
 
-#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || (!defined(BGEMM) && defined(SBGEMM_GEMV_FORWARD)) || (defined(BGEMM) && defined(BGEMM_GEMV_FORWARD)))
+#if (!defined(BFLOAT16) || (!defined(BGEMM) && defined(SBGEMM_GEMV_FORWARD)) || (defined(BGEMM) && defined(BGEMM_GEMV_FORWARD)))
+#define BFLOAT16_GEMM_GEMV_FORWARD 1
+#else
+#define BFLOAT16_GEMM_GEMV_FORWARD 0
+#endif
+#if (!defined(HFLOAT16) || (!defined(HGEMM) && defined(SHGEMM_GEMV_FORWARD)) || (defined(HGEMM) && defined(HGEMM_GEMV_FORWARD)))
+#define HFLOAT16_GEMM_GEMV_FORWARD 1
+#else
+#define HFLOAT16_GEMM_GEMV_FORWARD 0
+#endif
+
+#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && HFLOAT16_GEMM_GEMV_FORWARD && BFLOAT16_GEMM_GEMV_FORWARD
 #if defined(ARCH_ARM64)
   // The gemv kernels in arm64/{gemv_n.S,gemv_n_sve.c,gemv_t.S,gemv_t_sve.c}
   // perform poorly in certain circumstances. We use the following boolean

interface/gemm_batch_strided.c

@@ -0,0 +1,425 @@
+/*****************************************************************************
+Copyright (c) 2025, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of 
+      its contributors may be used to endorse or promote products 
+      derived from this software without specific prior written 
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+**********************************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "common.h"
+
+void openblas_warning(int verbose, const char * msg);
+
+#ifndef COMPLEX
+#ifdef XDOUBLE
+#define ERROR_NAME "QGEMM_BATCH_STRIDED "
+#elif defined(DOUBLE)
+#define ERROR_NAME "DGEMM_BATCH_STRIDED "
+#define GEMM_BATCH_THREAD dgemm_batch_thread
+#else
+#define ERROR_NAME "SGEMM_BATCH_STRIDED "
+#define GEMM_BATCH_THREAD sgemm_batch_thread
+#endif
+#else
+#ifdef XDOUBLE
+#define ERROR_NAME "XGEMM_BATCH_STRIDED "
+#elif defined(DOUBLE)
+#define ERROR_NAME "ZGEMM_BATCH_STRIDED "
+#define GEMM_BATCH_THREAD zgemm_batch_thread
+#else
+#define ERROR_NAME "CGEMM_BATCH_STRIDED "
+#define GEMM_BATCH_THREAD cgemm_batch_thread
+#endif
+#endif
+static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, BLASLONG) = {
+  GEMM_NN, GEMM_TN, GEMM_RN, GEMM_CN,
+  GEMM_NT, GEMM_TT, GEMM_RT, GEMM_CT,
+  GEMM_NR, GEMM_TR, GEMM_RR, GEMM_CR,
+  GEMM_NC, GEMM_TC, GEMM_RC, GEMM_CC,
+};
+
+#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE)
+#define USE_SMALL_MATRIX_OPT 1
+#else
+#define USE_SMALL_MATRIX_OPT 0
+#endif
+
+#if USE_SMALL_MATRIX_OPT
+#ifndef DYNAMIC_ARCH
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(table[idx]))
+#else
+#define SMALL_KERNEL_ADDR(table, idx) ((void *)(*(uintptr_t *)((char *)gotoblas + (size_t)(table[idx]))))
+#endif
+
+
+#ifndef COMPLEX
+static size_t gemm_small_kernel[] = {
+	GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, 0, 0,
+	GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, 0, 0,
+};
+
+
+static size_t gemm_small_kernel_b0[] = {
+	GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, 0, 0,
+	GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, 0, 0,
+};
+
+#define GEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel_b0, (idx))
+#define GEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, IFLOAT *, BLASLONG, FLOAT, IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(gemm_small_kernel, (idx))
+#else
+
+static size_t zgemm_small_kernel[] = {
+	GEMM_SMALL_KERNEL_NN, GEMM_SMALL_KERNEL_TN, GEMM_SMALL_KERNEL_RN, GEMM_SMALL_KERNEL_CN,
+	GEMM_SMALL_KERNEL_NT, GEMM_SMALL_KERNEL_TT, GEMM_SMALL_KERNEL_RT, GEMM_SMALL_KERNEL_CT,
+	GEMM_SMALL_KERNEL_NR, GEMM_SMALL_KERNEL_TR, GEMM_SMALL_KERNEL_RR, GEMM_SMALL_KERNEL_CR,
+	GEMM_SMALL_KERNEL_NC, GEMM_SMALL_KERNEL_TC, GEMM_SMALL_KERNEL_RC, GEMM_SMALL_KERNEL_CC,
+};
+
+static size_t zgemm_small_kernel_b0[] = {
+	GEMM_SMALL_KERNEL_B0_NN, GEMM_SMALL_KERNEL_B0_TN, GEMM_SMALL_KERNEL_B0_RN, GEMM_SMALL_KERNEL_B0_CN,
+	GEMM_SMALL_KERNEL_B0_NT, GEMM_SMALL_KERNEL_B0_TT, GEMM_SMALL_KERNEL_B0_RT, GEMM_SMALL_KERNEL_B0_CT,
+	GEMM_SMALL_KERNEL_B0_NR, GEMM_SMALL_KERNEL_B0_TR, GEMM_SMALL_KERNEL_B0_RR, GEMM_SMALL_KERNEL_B0_CR,
+	GEMM_SMALL_KERNEL_B0_NC, GEMM_SMALL_KERNEL_B0_TC, GEMM_SMALL_KERNEL_B0_RC, GEMM_SMALL_KERNEL_B0_CC,
+};
+
+#define ZGEMM_SMALL_KERNEL(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel, (idx))
+#define ZGEMM_SMALL_KERNEL_B0(idx) (int (*)(BLASLONG, BLASLONG, BLASLONG, FLOAT *, BLASLONG, FLOAT , FLOAT, FLOAT *, BLASLONG, FLOAT *, BLASLONG)) SMALL_KERNEL_ADDR(zgemm_small_kernel_b0, (idx))
+#endif
+#endif
+
+#ifndef CBLAS
+void NAME(char *transa, char *transb,
+	   blasint * M, blasint * N, blasint * K,
+	   FLOAT * Alpha,
+	   IFLOAT * a, blasint * Lda,
+	   blasint * stride_a,
+	   IFLOAT *b, blasint * Ldb,
+	   blasint * stride_b,
+	   FLOAT * Beta,
+	   FLOAT * c, blasint * Ldc, blasint * stride_c, blasint * matcount) {
+ 	  
+           char ta = *transa;
+   	   char tb = *transb;	   
+	   blasint count = *matcount;
+	   blasint stridea= *stride_a;
+	   blasint strideb= *stride_b;
+	   blasint stridec= *stride_c;
+	   blasint m=*M;
+	   blasint n=*N;
+	   blasint k=*K;
+	   blasint lda=*Lda;
+	   blasint ldb=*Ldb;
+	   blasint ldc=*Ldc;
+#if !defined(COMPLEX)	   
+	   FLOAT alpha=*Alpha;
+	   FLOAT beta=*Beta;
+#else
+	   FLOAT *alpha=Alpha;
+	   FLOAT *beta=Beta;
+#endif
+#else
+
+void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE transa, enum CBLAS_TRANSPOSE transb,
+	   blasint m, blasint n, blasint k,
+#ifndef COMPLEX
+	   FLOAT alpha,
+	   IFLOAT * a, blasint lda, blasint stridea,
+	   IFLOAT * b, blasint ldb, blasint strideb,
+	   FLOAT beta,
+	   FLOAT * c, blasint ldc, blasint stridec, blasint count) {
+#else
+	   void * valpha,
+	   void * va, blasint lda, blasint stridea,
+	   void * vb, blasint ldb, blasint strideb,
+	   void * vbeta,
+	   void * vc, blasint ldc, blasint stridec, blasint count) {
+
+  FLOAT * alpha=(FLOAT *)valpha;
+  FLOAT * beta=(FLOAT *)vbeta;
+  FLOAT * a=(FLOAT*)va;
+  FLOAT * b=(FLOAT*)vb;
+  FLOAT * c=(FLOAT*)vc;
+#endif
+#endif
+  BLASLONG group_m, group_n, group_k;
+  BLASLONG group_lda, group_ldb, group_ldc;
+
+  blas_arg_t * args_array=NULL;
+
+  int mode=0, group_mode=0;
+
+  blasint i=0;
+
+  int group_transa, group_transb;
+  BLASLONG group_nrowa, group_nrowb;
+  blasint info;
+
+  void * group_routine=NULL;
+#ifdef SMALL_MATRIX_OPT
+  void * group_small_matrix_opt_routine=NULL;
+#endif
+  
+#if defined (SMP) || defined(SMALL_MATRIX_OPT)
+  double MNK;
+#endif
+
+  PRINT_DEBUG_CNAME;
+
+  args_array=(blas_arg_t *)malloc(count * sizeof(blas_arg_t));
+  
+  if(args_array == NULL){
+    openblas_warning(0, "memory alloc failed!\n");
+    return;
+  }
+
+#ifdef SMP
+#ifndef COMPLEX
+#ifdef XDOUBLE
+  mode  =  BLAS_XDOUBLE | BLAS_REAL;
+#elif defined(DOUBLE)
+  mode  =  BLAS_DOUBLE  | BLAS_REAL;
+#else
+  mode  =  BLAS_SINGLE  | BLAS_REAL;
+#endif
+#else
+#ifdef XDOUBLE
+  mode  =  BLAS_XDOUBLE | BLAS_COMPLEX;
+#elif defined(DOUBLE)
+  mode  =  BLAS_DOUBLE  | BLAS_COMPLEX;
+#else
+  mode  =  BLAS_SINGLE  | BLAS_COMPLEX;
+#endif
+#endif
+#endif
+  
+  for(i=0; i<count; i++) {
+
+    group_m = group_n = group_k = 0;
+    group_lda = group_ldb = group_ldc = 0;
+    group_transa = -1;
+    group_transb = -1;
+    info   = 0;
+   
+#if defined(CBLAS) 
+    if (order == CblasColMajor) {
+      group_m = m;
+      group_n = n;
+      group_k = k;
+
+      
+      if (transa == CblasNoTrans)     group_transa = 0;
+      if (transa == CblasTrans)       group_transa = 1;
+#ifndef COMPLEX
+      if (transa == CblasConjNoTrans) group_transa = 0;
+      if (transa == CblasConjTrans)   group_transa = 1;
+#else
+      if (transa == CblasConjNoTrans) group_transa = 2;
+      if (transa == CblasConjTrans)   group_transa = 3;
+#endif
+      if (transb == CblasNoTrans)     group_transb = 0;
+      if (transb == CblasTrans)       group_transb = 1;
+#ifndef COMPLEX
+      if (transb == CblasConjNoTrans) group_transb = 0;
+      if (transb == CblasConjTrans)   group_transb = 1;
+#else
+      if (transb == CblasConjNoTrans) group_transb = 2;
+      if (transb == CblasConjTrans)   group_transb = 3;
+#endif
+      group_nrowa = group_m;
+      if (group_transa & 1) group_nrowa = group_k;
+      group_nrowb = group_k;
+      if (group_transb & 1) group_nrowb = group_n;
+
+      info=-1;
+	
+      if (group_ldc < group_m) info = 13;
+      if (group_ldb < group_nrowb)  info = 10;
+      if (group_lda < group_nrowa)  info =  8;
+      if (group_k < 0)        info =  5;
+      if (group_n < 0)        info =  4;
+      if (group_m < 0)        info =  3;
+      if (group_transb < 0)        info =  2;
+      if (group_transa < 0)        info =  1;
+
+    }else if (order == CblasRowMajor) {
+      
+      group_m = n;
+      group_n = m;
+      group_k = k;
+
+      group_lda = ldb;
+      group_ldb = lda;
+      group_ldc = ldc;
+
+      if (transb == CblasNoTrans)     group_transa = 0;
+      if (transb == CblasTrans)       group_transa = 1;
+#ifndef COMPLEX
+      if (transb == CblasConjNoTrans) group_transa = 0;
+      if (transb == CblasConjTrans)   group_transa = 1;
+#else
+      if (transb == CblasConjNoTrans) group_transa = 2;
+      if (transb == CblasConjTrans)   group_transa = 3;
+#endif
+      if (transa == CblasNoTrans)     group_transb = 0;
+      if (transa == CblasTrans)       group_transb = 1;
+#ifndef COMPLEX
+      if (transa == CblasConjNoTrans) group_transb = 0;
+      if (transa == CblasConjTrans)   group_transb = 1;
+#else
+      if (transa == CblasConjNoTrans) group_transb = 2;
+      if (transa == CblasConjTrans)   group_transb = 3;
+#endif
+
+#else
+      group_m = m;
+      group_n = n;
+      group_k = k;
+
+      group_lda = lda;
+      group_ldb = ldb;
+      group_ldc = ldc;
+      
+      if (tb == 'N')   group_transa = 0;
+      if (tb == 'T')   group_transa = 1;
+#ifndef COMPLEX
+      if (tb == 'C')   group_transa = 1;
+#else
+      if (tb == 'C')   group_transa = 3;
+#endif
+      if (ta == 'N')   group_transb = 0;
+      if (ta == 'T')   group_transb = 1;
+#ifndef COMPLEX
+      if (ta == 'C')   group_transb = 1;
+#else
+      if (ta == 'C')   group_transb = 3;
+#endif
+#endif
+
+      group_nrowa = group_m;
+      if (group_transa & 1) group_nrowa = group_k;
+      group_nrowb = group_k;
+      if (group_transb & 1) group_nrowb = group_n;
+
+      info=-1;
+	
+      if (group_ldc < group_m) info = 13;
+      if (group_ldb < group_nrowb)  info = 10;
+      if (group_lda < group_nrowa)  info =  8;
+      if (group_k < 0)        info =  5;
+      if (group_n < 0)        info =  4;
+      if (group_m < 0)        info =  3;
+      if (group_transb < 0)        info =  2;
+      if (group_transa < 0)        info =  1;      
+#if defined(CBLAS)
+    }
+#endif
+
+    if (info >= 0) {
+      BLASFUNC(xerbla)(ERROR_NAME, &info, sizeof(ERROR_NAME));
+      free(args_array);
+      return;
+    }
+
+    if (group_m == 0 || group_n == 0) continue;
+
+    group_mode=mode;
+
+#if defined(SMP) || defined(SMALL_MATRIX_OPT)
+    MNK = (double) group_m * (double) group_n * (double) group_k;
+#endif
+
+#ifdef SMALL_MATRIX_OPT
+    if (MNK <= 100.0*100.0*100.0){
+      group_routine=NULL;
+#if !defined(COMPLEX)
+      if(beta == 0.0){
+	group_mode=mode | BLAS_SMALL_B0_OPT;
+	group_small_matrix_opt_routine=(void *)(gemm_small_kernel_b0[(group_transb<<2)|group_transa]);
+      }else{
+	group_mode=mode | BLAS_SMALL_OPT;
+	group_small_matrix_opt_routine=(void *)(gemm_small_kernel[(group_transb<<2)|group_transa]);
+      }
+#else
+      if(beta[0] == 0.0 && beta[1] == 0.0){
+	group_mode=mode | BLAS_SMALL_B0_OPT;
+	group_small_matrix_opt_routine=(void *)(zgemm_small_kernel_b0[(group_transb<<2)|group_transa]);
+      }else{
+	group_mode=mode | BLAS_SMALL_OPT;
+	group_small_matrix_opt_routine=(void *)(zgemm_small_kernel[(group_transb<<2)|group_transa]);
+      }
+
+#endif
+      
+    }else{
+#endif
+      group_routine=(void*)(gemm[(group_transb<<2)|group_transa]);
+#ifdef SMALL_MATRIX_OPT
+    }
+#endif    
+
+    
+      args_array[i].m=group_m;
+      args_array[i].n=group_n;
+      args_array[i].k=group_k;
+      args_array[i].lda=group_lda;
+      args_array[i].ldb=group_ldb;
+      args_array[i].ldc=group_ldc;
+      args_array[i].alpha=&alpha;
+      args_array[i].beta=&beta;
+
+#if defined(CBLAS)
+      if (order == CblasColMajor) {      
+	args_array[i].a=&(a[i*stridea]);
+	args_array[i].b=&(b[i*strideb]);
+      }else if(order == CblasRowMajor){
+	args_array[i].a=&(b[i*strideb]);
+	args_array[i].b=&(a[i*stridea]);
+      }
+#else
+	args_array[i].a=&(a[i*stridea]);
+	args_array[i].b=&(b[i*strideb]);
+#endif
+      
+      args_array[i].c= &c[i*stridec];
+      
+      args_array[i].routine_mode=group_mode;
+      args_array[i].routine=group_routine;
+#ifdef SMALL_MATRIX_OPT
+      if (!group_routine)
+        args_array[i].routine=group_small_matrix_opt_routine;
+#endif      
+  }
+
+  if(count>0) {
+    GEMM_BATCH_THREAD(args_array,count);
+  }
+
+  free(args_array);
+}

interface/sbgemv.c

@@ -48,6 +48,10 @@
 #define GEMV_THREAD_N bgemv_thread_n
 #define GEMV_THREAD_T bgemv_thread_t
 #define ERROR_NAME "BGEMV "
+#elif defined(HFLOAT16)
+#define GEMV_THREAD_N shgemv_thread_n
+#define GEMV_THREAD_T shgemv_thread_t
+#define ERROR_NAME "SHGEMV "
 #else
 #define GEMV_THREAD_N sbgemv_thread_n
 #define GEMV_THREAD_T sbgemv_thread_t

interface/symm.c

@@ -371,6 +371,24 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_SIDE Side, enum CBLAS_UPLO Uplo,
     return;
   }
 
+#if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16)  && !defined(HFLOAT16)
+#if defined(ARCH_ARM64) && (defined(USE_SSYMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH))
+#if defined(DYNAMIC_ARCH)
+ if (support_sme1())
+#endif
+   if (args.m == 0 || args.n == 0) return;
+   if (order == CblasRowMajor && m == lda && n == ldb && n == ldc)
+   {
+     if (Side == CblasLeft && Uplo == CblasUpper) {
+       SSYMM_DIRECT_ALPHA_BETA_LU(m, n, alpha, a, lda, b, ldb, beta, c, ldc); return;
+     }
+     else if (Side == CblasLeft && Uplo == CblasLower) {
+       SSYMM_DIRECT_ALPHA_BETA_LL(m, n, alpha, a, lda, b, ldb, beta, c, ldc); return;
+     }
+   }
+#endif
+#endif
+
 #endif
 
   if (args.m == 0 || args.n == 0) return;

interface/syrk.c

@@ -338,6 +338,23 @@ double NNK;
     BLASFUNC(xerbla)(ERROR_NAME, &info, sizeof(ERROR_NAME));
     return;
   }
+#if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16)  && !defined(HFLOAT16)
+#if defined(ARCH_ARM64) && (defined(USE_SSYRK_KERNEL_DIRECT)||defined(DYNAMIC_ARCH))
+#if defined(DYNAMIC_ARCH)
+ if (support_sme1())
+#endif
+  if (args.n == 0) return;
+  if (order == CblasRowMajor && n == ldc) {
+    if (Trans == CblasNoTrans && k == lda) {
+      (Uplo == CblasUpper ? SSYRK_DIRECT_ALPHA_BETA_UN : SSYRK_DIRECT_ALPHA_BETA_LN)(n, k, alpha, a, lda, beta, c, ldc);
+      return;
+    } else if (Trans == CblasTrans && n == lda){
+      (Uplo == CblasUpper ? SSYRK_DIRECT_ALPHA_BETA_UT : SSYRK_DIRECT_ALPHA_BETA_LT)(n, k, alpha, a, lda, beta, c, ldc);
+      return;
+    }
+  }
+#endif
+#endif
 
 #endif
 

interface/trsm.c

@@ -355,6 +355,23 @@ void CNAME(enum CBLAS_ORDER order,
     return;
   }
 
+#if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16)  && !defined(HFLOAT16)
+#if defined(ARCH_ARM64) && (defined(USE_STRMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH))
+#if defined(DYNAMIC_ARCH)
+ if (support_sme1())
+#endif
+  if (args.m == 0 || args.n == 0) return;
+  if (order == CblasRowMajor && Diag == CblasNonUnit && Side == CblasLeft && m == lda && n == ldb) {
+    if (Trans ==  CblasNoTrans) {
+      (Uplo == CblasUpper ? STRMM_DIRECT_LNUN : STRMM_DIRECT_LNLN)(m, n, alpha, a, lda, b, ldb);
+    } else if (Trans == CblasTrans) {
+      (Uplo == CblasUpper ? STRMM_DIRECT_LTUN : STRMM_DIRECT_LTLN)(m, n, alpha, a, lda, b, ldb);
+    }
+    return;
+  }
+#endif
+#endif
+
 #endif
 
   if ((args.m == 0) || (args.n == 0)) return;

interface/zher.c

@@ -177,7 +177,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, blasint n, FLOAT alpha,
   buffer = (FLOAT *)blas_memory_alloc(1);
 
 #ifdef SMP
-  nthreads = num_cpu_avail(2);
+  if (n < 100)
+    nthreads = 1;
+  else
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif

interface/zher2.c

@@ -186,7 +186,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, blasint n, void *VALPHA
   buffer = (FLOAT *)blas_memory_alloc(1);
 
 #ifdef SMP
-  nthreads = num_cpu_avail(2);
+  if (n < 100)
+	nthreads = 1;
+  else
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif

interface/zhpr.c

@@ -175,7 +175,10 @@ void CNAME(enum CBLAS_ORDER order,
   buffer = (FLOAT *)blas_memory_alloc(1);
 
 #ifdef SMP
-  nthreads = num_cpu_avail(2);
+  if (n < 100)
+    nthreads = 1;
+  else
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif

interface/zhpr2.c

@@ -187,7 +187,10 @@ void CNAME(enum CBLAS_ORDER order,
   buffer = (FLOAT *)blas_memory_alloc(1);
 
 #ifdef SMP
-  nthreads = num_cpu_avail(2);
+  if (n < 100)
+	nthreads = 1;
+  else
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif

kernel/CMakeLists.txt

@@ -228,6 +228,10 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16")
     endif ()
+    if (BUILD_HFLOAT16)
+	    GenerateNamedObjects("${KERNELDIR}/${SHGEMVNKERNEL}" "" "gemv_n" false "" "" false "HFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${SHGEMVTKERNEL}" "" "gemv_t" false "" "" false "HFLOAT16")
+    endif ()
     # Makefile.L3
     set(USE_TRMM false)
     string(TOUPPER ${TARGET_CORE} UC_TARGET_CORE)
@@ -237,10 +241,22 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     if (ZARCH OR (UC_TARGET_CORE MATCHES POWER8) OR (UC_TARGET_CORE MATCHES POWER9) OR (UC_TARGET_CORE MATCHES POWER10))
       set(USE_TRMM true)
     endif ()
+    set(USE_DIRECT_STRMM false)
+    if (ARM64)
+       set(USE_DIRECT_STRMM true)
+    endif()
+    set(USE_DIRECT_SSYRK false)
+    if (ARM64)
+       set(USE_DIRECT_SSYRK true)
+    endif()
     set(USE_DIRECT_SGEMM false)
     if (X86_64 OR ARM64)
        set(USE_DIRECT_SGEMM true)
     endif()
+    set(USE_DIRECT_SSYMM false)
+    if (ARM64)
+       set(USE_DIRECT_SSYMM true)
+    endif()
     if (UC_TARGET_CORE MATCHES ARMV9SME)
           set (HAVE_SME true)
     endif ()
@@ -267,6 +283,34 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
    	  endif ()
     endif()
 
+    if (USE_DIRECT_SSYMM)
+      if (ARM64)
+	  set (SSYMMDIRECTKERNEL_ALPHA_BETA ssymm_direct_alpha_beta_arm64_sme1.c)
+      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLU" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLL" false "" "" false SINGLE)
+   	  endif ()
+    endif()
+
+    if (USE_DIRECT_STRMM)
+        if (ARM64)
+        set (STRMMDIRECTKERNEL strmm_direct_arm64_sme1.c)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNUN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNLN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTUN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTLN" false "" "" false SINGLE)
+        endif ()
+      endif ()
+
+    if (USE_DIRECT_SSYRK)
+      if (ARM64)
+      set (SSYRKDIRECTKERNEL_ALPHA_BETA ssyrk_direct_alpha_beta_arm64_sme1.c)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUN" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUT" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLN" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLT" false "" "" false SINGLE)
+      endif ()
+    endif()
+
     foreach (float_type SINGLE DOUBLE)
       string(SUBSTRING ${float_type} 0 1 float_char)
       GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type})
@@ -442,6 +486,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
         set(TRMM_KERNEL "${${float_char}GEMMKERNEL}")
       endif ()
 
+
       if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
 
         # just enumerate all these. there is an extra define for these indicating which side is a conjugate (e.g. CN NC NN) that I don't really want to work into GenerateCombinationObjects

kernel/Makefile.L2

@@ -101,6 +101,16 @@ SBGEMVTKERNEL = ../x86_64/sbgemv_t.c
 endif
 endif
 
+ifeq ($(BUILD_HFLOAT16),1)
+ifndef SHGEMVNKERNEL
+SHGEMVNKERNEL = ../generic/gemv_n.c
+endif
+
+ifndef SHGEMVTKERNEL
+SHGEMVTKERNEL = ../generic/gemv_t.c
+endif
+endif
+
 ### GER ###
 
 ifndef SGERKERNEL
@@ -299,6 +309,12 @@ SBBLASOBJS     += \
         sbgemv_t$(TSUFFIX).$(SUFFIX)
 endif
 
+ifeq ($(BUILD_HFLOAT16),1)
+SHBLASOBJS     += \
+        shgemv_n$(TSUFFIX).$(SUFFIX) \
+        shgemv_t$(TSUFFIX).$(SUFFIX)
+endif
+
 ifneq "$(or $(BUILD_SINGLE), $(BUILD_DOUBLE), $(BUILD_COMPLEX))" ""
 $(KDIR)sgemv_n$(TSUFFIX).$(SUFFIX)  $(KDIR)sgemv_n$(TSUFFIX).$(PSUFFIX)  : $(KERNELDIR)/$(SGEMVNKERNEL) $(TOPDIR)/common.h $(GEMVDEP)
 	$(CC) -c $(CFLAGS) -UDOUBLE -UCOMPLEX  -UTRANS $< -o $@
@@ -558,3 +574,10 @@ $(KDIR)bgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERN
 	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@
 endif
 
+ifeq ($(BUILD_HFLOAT16),1)
+$(KDIR)shgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)shgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SHGEMVNKERNEL)
+	$(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@
+$(KDIR)shgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)shgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SHGEMVTKERNEL)
+	$(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@
+endif
+

kernel/Makefile.L3

@@ -52,6 +52,9 @@ endif
 ifeq ($(ARCH), arm64)
 USE_TRMM = 1
 USE_DIRECT_SGEMM = 1
+USE_DIRECT_SSYMM = 1
+USE_DIRECT_STRMM = 1
+USE_DIRECT_SSYRK = 1
 endif
 
 ifeq ($(ARCH), riscv64)
@@ -137,6 +140,39 @@ endif
 endif
 endif
 
+ifdef USE_DIRECT_SSYMM
+ifndef SSYMMDIRECTKERNEL_ALPHA_BETA
+ifeq ($(ARCH), arm64)
+ifeq ($(TARGET_CORE), ARMV9SME)
+HAVE_SME = 1
+endif
+SSYMMDIRECTKERNEL_ALPHA_BETA = ssymm_direct_alpha_beta_arm64_sme1.c
+endif
+endif
+endif
+
+ifdef USE_DIRECT_STRMM
+ifndef STRMMDIRECTKERNEL
+ifeq ($(ARCH), arm64)
+ifeq ($(TARGET_CORE), ARMV9SME)
+HAVE_SME = 1
+endif
+STRMMDIRECTKERNEL = strmm_direct_arm64_sme1.c
+endif
+endif
+endif
+
+ifdef USE_DIRECT_SSYRK
+ifndef SSYRKDIRECTKERNEL_ALPHA_BETA
+ifeq ($(ARCH), arm64)
+ifeq ($(TARGET_CORE), ARMV9SME)
+HAVE_SME = 1
+endif
+SSYRKDIRECTKERNEL_ALPHA_BETA = ssyrk_direct_alpha_beta_arm64_sme1.c
+endif
+endif
+endif
+
 ifeq ($(BUILD_BFLOAT16), 1)
 ifndef BGEMMKERNEL
 BGEMM_BETA = ../generic/gemm_beta.c
@@ -220,6 +256,30 @@ endif
 endif
 endif
 
+ifdef USE_DIRECT_SSYMM
+ifeq ($(ARCH), arm64)
+SKERNELOBJS += \
+	ssymm_direct_alpha_betaLU$(TSUFFIX).$(SUFFIX) \
+	ssymm_direct_alpha_betaLL$(TSUFFIX).$(SUFFIX)
+endif
+endif
+
+ifdef USE_DIRECT_STRMM
+ifeq ($(ARCH), arm64)
+SKERNELOBJS += \
+	strmm_direct_LNUN$(TSUFFIX).$(SUFFIX) strmm_direct_LNLN$(TSUFFIX).$(SUFFIX) \
+	strmm_direct_LTUN$(TSUFFIX).$(SUFFIX) strmm_direct_LTLN$(TSUFFIX).$(SUFFIX)
+endif
+endif
+
+ifdef USE_DIRECT_SSYRK
+ifeq ($(ARCH), arm64)
+SKERNELOBJS += \
+	ssyrk_direct_alpha_betaUN$(TSUFFIX).$(SUFFIX) ssyrk_direct_alpha_betaUT$(TSUFFIX).$(SUFFIX) \
+	ssyrk_direct_alpha_betaLN$(TSUFFIX).$(SUFFIX) ssyrk_direct_alpha_betaLT$(TSUFFIX).$(SUFFIX)
+endif
+endif
+
 ifneq "$(or $(BUILD_DOUBLE),$(BUILD_COMPLEX16))" ""
 DKERNELOBJS	+= \
 	dgemm_beta$(TSUFFIX).$(SUFFIX) \
@@ -982,6 +1042,15 @@ endif
 endif
 endif
 
+ifdef USE_DIRECT_SSYMM
+ifeq ($(ARCH), arm64)
+$(KDIR)ssymm_direct_alpha_betaLU$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYMMDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DLEFT -DUPPER $< -o $@
+$(KDIR)ssymm_direct_alpha_betaLL$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYMMDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DLEFT -DLOWER $< -o $@
+endif
+endif
+
 ifeq ($(BUILD_BFLOAT16), 1)
 $(KDIR)bgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) 
 	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
@@ -1108,6 +1177,21 @@ $(KDIR)xgemm_kernel_r$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMD
 $(KDIR)xgemm_kernel_b$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(XGEMMKERNEL) $(XGEMMDEPEND)
 	$(CC) $(CFLAGS) -c -DXDOUBLE -DCOMPLEX -DCC $< -o $@
 
+ifdef USE_DIRECT_SSYRK
+ifeq ($(ARCH), arm64)
+$(KDIR)ssyrk_direct_alpha_betaUN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYRKDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DUPPER -UTRANSA $< -o $@
+
+$(KDIR)ssyrk_direct_alpha_betaUT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYRKDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DUPPER -DTRANSA $< -o $@
+
+$(KDIR)ssyrk_direct_alpha_betaLN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYRKDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -UUPPER -UTRANSA $< -o $@
+
+$(KDIR)ssyrk_direct_alpha_betaLT$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SSYRKDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -UUPPER -DTRANSA $< -o $@
+endif
+endif
 
 ifdef USE_TRMM
 $(KDIR)strmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMKERNEL)
@@ -1150,6 +1234,23 @@ else
 	$(CC) $(CFLAGS) -c -DTRMMKERNEL -UDOUBLE -UCOMPLEX -ULEFT -DTRANSA $< -o $@
 endif
 
+
+ifdef USE_DIRECT_STRMM
+ifeq ($(ARCH), arm64)
+$(KDIR)strmm_direct_LNUN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMDIRECTKERNEL)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -UTRANSA -DUPPER $< -o $@
+
+$(KDIR)strmm_direct_LNLN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMDIRECTKERNEL)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -UTRANSA -UUPPER $< -o $@
+
+$(KDIR)strmm_direct_LTUN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMDIRECTKERNEL)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DTRANSA -DUPPER $< -o $@
+
+$(KDIR)strmm_direct_LTLN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(STRMMDIRECTKERNEL)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX -DTRANSA -UUPPER $< -o $@
+endif
+endif
+
 $(KDIR)dtrmm_kernel_LN$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DTRMMKERNEL)
 ifeq ($(OS), AIX)
 	$(CC) $(CFLAGS) -S -DTRMMKERNEL -DDOUBLE -UCOMPLEX -DLEFT -UTRANSA $< -o - > dtrmm_kernel_ln.s

kernel/arm64/KERNEL.A64FX

@@ -5,8 +5,11 @@ DGEMVNKERNEL = gemv_n_sve_v4x3.c
 SGEMVTKERNEL = gemv_t_sve_v4x3.c
 DGEMVTKERNEL = gemv_t_sve_v4x3.c
 
-DDOTKERNEL   = dot_sve_v8.c
-SDOTKERNEL   = dot_sve_v8.c
+DDOTKERNEL   = dot.c
+SDOTKERNEL   = dot.c
 
 SAXPYKERNEL  = axpy_sve.c
 DAXPYKERNEL  = axpy_sve.c
+
+SGERKERNEL   = ger_sve_v1x3.c
+DGERKERNEL   = ger_sve_v1x3.c

kernel/arm64/KERNEL.NEOVERSEN2

@@ -189,7 +189,7 @@ ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16), 1)
-BGEMM_BETA    =  sbgemm_beta_neoversen2.c
+BGEMM_BETA    =  bgemm_beta_neon.c
 BGEMMKERNEL    = sbgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversen2.c
 BGEMMINCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_M)_neoversen2.c
 BGEMMITCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_M)_neoversen2.c

kernel/arm64/KERNEL.NEOVERSEV1

@@ -70,3 +70,6 @@ SBGEMVNKERNEL = sbgemv_n_neon.c
 SBGEMVTKERNEL = sbgemv_t_bfdot.c
 
 endif
+
+SGERKERNEL   = ger_sve_v1x3.c
+DGERKERNEL   = ger_sve_v1x3.c

kernel/arm64/bgemm_kernel_2vlx4_neoversev1_impl.c

@@ -40,7 +40,8 @@
 
 #define UPDATE_C(PG, PTR, DST, SRC)                                            \
   do {                                                                         \
-    DST = svreinterpret_f32_u32(svld1uh_u32((pghalf), (uint16_t*)PTR));        \
+    svtmp16 = svld1_bf16((pghalf), (PTR));                                     \
+    DST = svreinterpret_f32(svzip1_bf16(zeros, svtmp16));                      \
     DST = svadd_z((PG), SRC, DST);                                             \
     svtmp16 = svcvt_bf16_f32_z((PG), DST);                                     \
     svtmp16 = svuzp1_bf16(svtmp16, svtmp16);                                   \
@@ -55,7 +56,8 @@
 
 #define UPDATE_C(PG, PTR, DST, SRC)                                            \
   do {                                                                         \
-    DST = svreinterpret_f32_u32(svld1uh_u32((pghalf), (uint16_t*)PTR));        \
+    svtmp16 = svld1_bf16((pghalf), (PTR));                                     \
+    DST = svreinterpret_f32(svzip1_bf16(zeros, svtmp16));                      \
     DST = svmad_z((PG), svalpha, SRC, DST);                                    \
     svtmp16 = svcvt_bf16_f32_z((PG), DST);                                     \
     svtmp16 = svuzp1_bf16(svtmp16, svtmp16);                                   \
@@ -133,6 +135,7 @@ static int bgemm_kernel_neoversev1_alpha(BLASLONG m, BLASLONG n, BLASLONG k,
     OUTPUT_FLOAT *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3;
     svfloat32_t tmp0, tmp1, tmp2, tmp3;
 #ifdef BGEMM
+    svbfloat16_t zeros = svdup_n_bf16(TO16(0.0));
     svbfloat16_t svtmp16;
 #else
     float32x2_t tmp4, tmp5, tmp6, tmp7;

kernel/arm64/dot.c

@@ -42,6 +42,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef USE_SVE
 #ifdef DOT_KERNEL_SVE
 #include DOT_KERNEL_SVE
+#elif defined(A64FX)
+#include "dot_kernel_sve_v8.c"
 #else
 #include "dot_kernel_sve.c"
 #endif
@@ -82,14 +84,43 @@ static inline int get_dot_optimal_nthreads_neoversev1(BLASLONG N, int ncpu) {
 }
 #endif
 
+#if defined(DYNAMIC_ARCH) || defined(A64FX)
+static inline int get_dot_optimal_nthreads_a64fx(BLASLONG N, int ncpu) {
+  #ifdef DOUBLE
+  return (N <= 11000L) ? 1 
+      : (N <= 20000L)  ? MIN(ncpu, 2) 
+      : (N <= 35000L)  ? MIN(ncpu, 4) 
+      : (N <= 50000L)  ? MIN(ncpu, 6) 
+      : (N <= 440000L)  ? MIN(ncpu, 8) 
+      : (N <= 880000L)  ? MIN(ncpu, 16) 
+      : (N <= 1020000L) ? MIN(ncpu, 24) 
+      : ncpu;
+  #else
+  return (N <= 22000L) ? 1 
+      : (N <= 39000L)  ? MIN(ncpu, 2) 
+      : (N <= 79000L)  ? MIN(ncpu, 4) 
+      : (N <= 120000L)  ? MIN(ncpu, 6) 
+      : (N <= 1020000L)  ? MIN(ncpu, 8) 
+      : ncpu;
+  #endif
+}
+#endif
+
 static inline int get_dot_optimal_nthreads(BLASLONG n) {
   int ncpu = num_cpu_avail(1);
 
-#if defined(NEOVERSEV1) && !defined(COMPLEX) && !defined(BFLOAT16)
+#if defined(A64FX) && !defined(COMPLEX) && !defined(BFLOAT16)
+  return get_dot_optimal_nthreads_a64fx(n, ncpu);
+#elif defined(NEOVERSEV1) && !defined(COMPLEX) && !defined(BFLOAT16)
   return get_dot_optimal_nthreads_neoversev1(n, ncpu);
 #elif defined(DYNAMIC_ARCH) && !defined(COMPLEX) && !defined(BFLOAT16)
-  if (strcmp(gotoblas_corename(), "neoversev1") == 0) {
-    return get_dot_optimal_nthreads_neoversev1(n, ncpu);
+  {
+    const char *core = gotoblas_corename();
+    if (strcmp(core, "a64fx") == 0) {
+      return get_dot_optimal_nthreads_a64fx(n, ncpu);
+    } else if (strcmp(core, "neoversev1") == 0) {
+      return get_dot_optimal_nthreads_neoversev1(n, ncpu);
+    }
   }
 #endif
 
@@ -97,7 +128,7 @@ static inline int get_dot_optimal_nthreads(BLASLONG n) {
   if (n <= 10000L)
     return 1;
   else
-    return num_cpu_avail(1);
+    return ncpu;
 }
 #endif
 

kernel/arm64/dot_kernel_sve.c

@@ -48,7 +48,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 "        add     x12, %[X_], x9, lsl #"SHIFT"               \n" \
 "        add     x13, %[Y_], x9, lsl #"SHIFT"               \n"
 #define TAIL_WHILE                                        \
-"        whilelo p1."DTYPE", x8, x0                         \n"
+"        whilelo p1."DTYPE", x8, %[N_]                         \n"
 #define UPDATE(pg, x,y,out)                               \
 "        ld1"WIDTH"    { z2."DTYPE" }, "pg"/z, ["x", x8, lsl #"SHIFT"]  \n" \
 "        ld1"WIDTH"    { z3."DTYPE" }, "pg"/z, ["y", x8, lsl #"SHIFT"]  \n" \
@@ -66,7 +66,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 "        movi    d1, #0x0                             \n" \
         SETUP_TRUE                                        \
 "        neg     x10, x9, lsl #1                      \n" \
-"        ands    x11, x10, x0                         \n" \
+"        ands    x11, x10, %[N_]                      \n" \
 "        b.eq    2f // skip_2x                        \n" \
         OFFSET_INPUTS                                     \
 "1: // vector_2x                                      \n" \
@@ -78,7 +78,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         SUM_VECTOR("1") \
 "2: // skip_2x                                        \n" \
 "        neg     x10, x9                              \n" \
-"        and     x10, x10, x0                         \n" \
+"        and     x10, x10, %[N_]                      \n" \
 "        cmp     x8, x10                              \n" \
 "        b.hs    4f // tail                           \n" \
 "3: // vector_1x                                      \n" \
@@ -87,7 +87,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 "        cmp     x8, x10                              \n" \
 "        b.lo    3b // vector_1x                      \n" \
 "4: // tail                                           \n" \
-"        cmp     x10, x0                              \n" \
+"        cmp     x10, %[N_]                           \n" \
 "        b.eq    5f // end                            \n" \
         TAIL_WHILE                                        \
         UPDATE("p1", "%[X_]", "%[Y_]", "z0")              \
@@ -110,9 +110,9 @@ dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y)
         [Y_] "r" (y)
       : "cc",
         "memory",
-        "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
-        "x8", "x9", "x10", "x11", "x12", "x13", "d1", 
-        "z0", "z1"
+        "x8", "x9", "x10", "x11", "x12", "x13",
+        "v0", "v1",
+        "z0", "z1", "z2", "z3"
   );
 
   return ret;

kernel/arm64/ger_sve_v1x3.c

@@ -0,0 +1,135 @@
+/***************************************************************************
+Copyright (c) 2025, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of
+      its contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <arm_sve.h>
+#include "common.h"
+
+#ifdef DOUBLE
+#define SV_COUNT svcntd
+#define SV_TYPE svfloat64_t
+#define SV_TRUE svptrue_b64
+#define SV_WHILE svwhilelt_b64_s64
+#define SV_DUP svdup_f64
+#else
+#define SV_COUNT svcntw
+#define SV_TYPE svfloat32_t
+#define SV_TRUE svptrue_b32
+#define SV_WHILE svwhilelt_b32_s64
+#define SV_DUP svdup_f32
+#endif
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha,
+	 FLOAT *x, BLASLONG incx,
+	 FLOAT *y, BLASLONG incy,
+	 FLOAT *a, BLASLONG lda, FLOAT *buffer){
+
+  FLOAT *X    = x;
+
+  if (incx != 1) {
+    X = buffer;
+    COPY_K(m, x, incx, X, 1);
+  }
+
+  BLASLONG width = (n + 3 - 1) / 3;
+  BLASLONG i, j;
+  BLASLONG sve_size = SV_COUNT();
+
+  FLOAT *y0_ptr = y + incy * width * 0;
+  FLOAT *y1_ptr = y + incy * width * 1;
+  FLOAT *y2_ptr = y + incy * width * 2;
+
+  for (j = 0; j < width; j++) {
+    svbool_t pg00 = (j + width * 0 < n) ? SV_TRUE() : svpfalse();
+    svbool_t pg01 = (j + width * 1 < n) ? SV_TRUE() : svpfalse();
+    svbool_t pg02 = (j + width * 2 < n) ? SV_TRUE() : svpfalse();
+
+    SV_TYPE temp0_vec = (j + width * 0 < n) ? SV_DUP(alpha * *y0_ptr) : SV_DUP(0.0);
+    SV_TYPE temp1_vec = (j + width * 1 < n) ? SV_DUP(alpha * *y1_ptr) : SV_DUP(0.0);
+    SV_TYPE temp2_vec = (j + width * 2 < n) ? SV_DUP(alpha * *y2_ptr) : SV_DUP(0.0);
+
+    FLOAT *x_ptr = X;
+    FLOAT *a0_ptr = a + lda * width * 0 + lda * j;
+    FLOAT *a1_ptr = a + lda * width * 1 + lda * j;
+    FLOAT *a2_ptr = a + lda * width * 2 + lda * j;
+
+    i = 0;
+    while (i + sve_size * 1 - 1 < m) {
+      SV_TYPE x0_vec = svld1_vnum(SV_TRUE(), x_ptr, 0);
+
+      SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr, 0);
+      SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr, 0);
+      SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr, 0);
+
+      a00_vec = svmla_x(pg00, a00_vec, temp0_vec, x0_vec);
+      a01_vec = svmla_x(pg01, a01_vec, temp1_vec, x0_vec);
+      a02_vec = svmla_x(pg02, a02_vec, temp2_vec, x0_vec);
+
+      svst1_vnum(pg00, a0_ptr, 0, a00_vec);
+      svst1_vnum(pg01, a1_ptr, 0, a01_vec);
+      svst1_vnum(pg02, a2_ptr, 0, a02_vec);
+
+      i += sve_size * 1;
+      x_ptr += sve_size * 1;
+      a0_ptr += sve_size * 1;
+      a1_ptr += sve_size * 1;
+      a2_ptr += sve_size * 1;
+    }
+
+    if (i < m) {
+      svbool_t pg0 = SV_WHILE(i + sve_size * 0, m);
+
+      pg00 = svand_z(SV_TRUE(), pg0, pg00);
+      pg01 = svand_z(SV_TRUE(), pg0, pg01);
+      pg02 = svand_z(SV_TRUE(), pg0, pg02);
+
+      SV_TYPE x0_vec = svld1_vnum(pg0, x_ptr, 0);
+
+      SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr, 0);
+      SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr, 0);
+      SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr, 0);
+
+      a00_vec = svmla_x(pg00, a00_vec, temp0_vec, x0_vec);
+      a01_vec = svmla_x(pg01, a01_vec, temp1_vec, x0_vec);
+      a02_vec = svmla_x(pg02, a02_vec, temp2_vec, x0_vec);
+
+      svst1_vnum(pg00, a0_ptr, 0, a00_vec);
+      svst1_vnum(pg01, a1_ptr, 0, a01_vec);
+      svst1_vnum(pg02, a2_ptr, 0, a02_vec);
+    }
+
+    y0_ptr += incy;
+    y1_ptr += incy;
+    y2_ptr += incy;
+  }
+  
+  return 0;
+}

kernel/arm64/sbgemm_kernel_8x4_neoversen2_impl.c

@@ -51,7 +51,8 @@
 #ifdef ALPHA_ONE
 #define UPDATE_C(PG16, PG32, PTR, SRC)                                    \
   do {                                                                    \
-    tmp32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR));   \
+    tmp16 = svld1_bf16((PG16), (PTR));                                    \
+    tmp32 = svreinterpret_f32(svzip1_bf16(zeros, tmp16));                 \
     tmp32 = svadd_z((PG32), SRC, tmp32);                                  \
     tmp16 = svcvt_bf16_f32_z((PG32), tmp32);                              \
     tmp16 = svuzp1_bf16(tmp16, tmp16);                                    \
@@ -60,7 +61,8 @@
 #else
 #define UPDATE_C(PG16, PG32, PTR, SRC)                                     \
   do {                                                                     \
-    tmp32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR));    \
+    tmp16 = svld1_bf16((PG16), (PTR));                                     \
+    tmp32 = svreinterpret_f32(svzip1_bf16(zeros, tmp16));                  \
     tmp32 = svmad_z((PG32), svalpha, SRC, tmp32);                          \
     tmp16 = svcvt_bf16_f32_z((PG32), tmp32);                               \
     tmp16 = svuzp1_bf16(tmp16, tmp16);                                     \
@@ -121,6 +123,7 @@ static int gemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOA
 #ifdef BGEMM
   svbool_t pg16_first_2 = svdupq_b16(1, 1, 0, 0, 0, 0, 0, 0);
   svbool_t pg16_first_1 = svdupq_b16(1, 0, 0, 0, 0, 0, 0, 0);
+  svbfloat16_t zeros = svdup_n_bf16(vcvth_bf16_f32(0.0));
 #endif
 
   bfloat16_t *ptr_a = (bfloat16_t *)A;

kernel/arm64/ssymm_direct_alpha_beta_arm64_sme1.c

@@ -0,0 +1,229 @@
+/*
+ Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ SPDX-License-Identifier: BSD-3-Clause-Clear
+*/
+
+#include "common.h"
+#include <stdlib.h>
+#include <inttypes.h>
+#include <math.h>
+// #include "sme_abi.h"
+#if defined(HAVE_SME)
+
+#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16
+#include <arm_sme.h>
+#endif
+
+/* Function prototypes */
+extern void sgemm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,\
+                                  const float * restrict a, float *  a_mod) __asm__("sgemm_direct_sme1_preprocess");
+
+extern void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\
+                                   const float *ba, const float *restrict bb, const float* beta,\
+                                   float *restrict C);
+/* Function Definitions */
+static uint64_t sve_cntw() {
+    uint64_t cnt;
+    asm volatile(
+        "rdsvl  %[res], #1\n"
+        "lsr    %[res], %[res], #2\n"
+        : [res] "=r" (cnt) ::
+    );
+    return cnt;
+}
+
+#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16
+
+__arm_new("za") __arm_locally_streaming
+static void ssymm_direct_sme1_preprocessLU(uint64_t nbr, uint64_t nbc,
+                const float *restrict a, float *restrict a_mod)
+{
+  // const uint64_t num_rows = nbr;
+  // const uint64_t num_cols = nbc;
+  const uint64_t svl = svcntw();
+  uint64_t row_batch = svl;
+
+  float *restrict pSrc;
+  float *restrict pDst;
+  for (uint64_t row_idx = 0; row_idx < nbr; row_idx += row_batch)
+  {
+    row_batch = MIN(row_batch, nbr - row_idx);
+
+    // Fill in the lower triangle and Transpose 1SVL x N panel of A
+    uint64_t col_batch = svl;
+
+    for (uint64_t col_idx = 0; col_idx < nbc; col_idx += col_batch)
+    {
+      svzero_za();
+
+      if (col_idx == row_idx)
+      {
+        pSrc = &a[(row_idx)*nbc + col_idx];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices, filling lower elements
+        const svbool_t pg_row = svwhilelt_b32_u64(col_idx, nbc);
+        for (int64_t row = row_batch - 1; row >= 0; row--)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+          svld1_ver_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < col_batch; col++)
+        {
+          svst1_ver_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+      else if (col_idx > row_idx)
+      {
+        pSrc = &a[(row_idx)*nbc + col_idx];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices
+        const svbool_t pg_row = svwhilelt_b32_u64(col_idx, nbc);
+        for (uint64_t row = 0; row < row_batch; row++)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < col_batch; col++)
+        {
+          svst1_ver_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+      else if (col_idx < row_idx)
+      {
+        pSrc = &a[row_idx + col_idx * nbc];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices
+        const svbool_t pg_row = svwhilelt_b32_u64(row_idx, nbc);
+        for (uint64_t row = 0; row < svl; row++)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < svl; col++)
+        {
+          svst1_hor_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+    }
+  }
+}
+
+//
+__arm_new("za") __arm_locally_streaming
+static void ssymm_direct_sme1_preprocessLL(uint64_t nbr, uint64_t nbc,
+                const float *restrict a, float *restrict a_mod)
+{
+  // const uint64_t num_rows = nbr;
+  const uint64_t svl = svcntw();
+  uint64_t row_batch = svl;
+
+  float *restrict pSrc;
+  float *restrict pDst;
+  for (uint64_t row_idx = 0; row_idx < nbr; row_idx += row_batch)
+  {
+    row_batch = MIN(row_batch, nbr - row_idx);
+
+    // Fill in the upper triangle and Transpose 1SVL x N panel of A
+    uint64_t col_batch = svl;
+
+    for (uint64_t col_idx = 0; col_idx < nbc; col_idx += col_batch)
+    {
+      svzero_za();
+
+      if (col_idx == row_idx)
+      {
+        pSrc = &a[(row_idx)*nbc + col_idx];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices, filling upper elements
+        const svbool_t pg_row = svwhilelt_b32_u64(col_idx, nbc);
+        for (uint64_t row = 0; row < row_batch; row++)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+          svld1_ver_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < col_batch; col++)
+        {
+          svst1_ver_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+      else if (col_idx > row_idx)
+      {
+        pSrc = &a[row_idx + col_idx * nbc];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices
+        const svbool_t pg_row = svptrue_b32();
+        for (uint64_t row = 0; row < row_batch; row++)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < col_batch; col++)
+        {
+          svst1_hor_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+      else if (col_idx < row_idx)
+      {
+        pSrc = &a[(row_idx)*nbc + col_idx];
+        pDst = &a_mod[(col_idx)*svl + row_idx * nbc];
+        // Load horizontal slices
+        const svbool_t pg_row = svwhilelt_b32_u64(col_idx, nbc);
+        for (uint64_t row = 0; row < row_batch; row++)
+        {
+          svld1_hor_za32(0, row, pg_row, &pSrc[row * nbc]);
+        }
+        // Save vertical slices
+        col_batch = MIN(col_batch, nbc - col_idx);
+        for (uint64_t col = 0; col < col_batch; col++)
+        {
+          svst1_ver_za32(0, col, svptrue_b32(), &pDst[col * svl]);
+        }
+      }
+    }
+  }
+}
+#else
+static void ssymm_direct_sme1_preprocessLU(uint64_t nbr, uint64_t nbc,
+                const float *restrict a, float *restrict a_mod){}
+static void ssymm_direct_sme1_preprocessLL(uint64_t nbr, uint64_t nbc,
+                const float *restrict a, float *restrict a_mod){}
+#endif
+
+//
+void CNAME(BLASLONG M, BLASLONG N, float alpha, float *__restrict A,
+           BLASLONG strideA, float *__restrict B, BLASLONG strideB,
+           float beta, float *__restrict R, BLASLONG strideR)
+{
+  uint64_t vl_elms = sve_cntw(); // vl_elem = 16
+  uint64_t m_mod = ceil((double)M / (double)vl_elms) * vl_elms;
+
+  /* Pre-process the left matrix to make it suitable for
+     matrix sum of outer-product calculation
+   */
+  float *A_mod = (float *)malloc(m_mod * M * sizeof(float));
+
+#if defined(UPPER)
+  ssymm_direct_sme1_preprocessLU(M, M, A, A_mod);
+#elif defined(LOWER)
+  ssymm_direct_sme1_preprocessLL(M, M, A, A_mod);
+#endif
+
+  /* Calculate C = alpha*A*B + beta*C */
+  sgemm_direct_alpha_beta_sme1_2VLx2VL(M, M, N, &alpha, A_mod, B, &beta, R);
+  free(A_mod);
+}
+
+#else
+
+void CNAME (BLASLONG M, BLASLONG N, float alpha, float * __restrict A,\
+            BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\
+            float beta, float * __restrict R, BLASLONG strideR){}
+ 
+#endif

kernel/arm64/ssyrk_direct_alpha_beta_arm64_sme1.c

@@ -0,0 +1,250 @@
+/*
+ Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ SPDX-License-Identifier: BSD-3-Clause-Clear
+*/
+
+#include "common.h"
+#include <stdlib.h>
+#include <inttypes.h>
+#include <math.h>
+#if defined(HAVE_SME)
+
+#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16
+#include <arm_sme.h>
+#endif
+
+/* Function prototypes */
+extern void sgemm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,\
+                                  const float * restrict a, float *  a_mod) __asm__("sgemm_direct_sme1_preprocess");
+
+/* Function Definitions */
+static uint64_t sve_cntw() {
+    uint64_t cnt;
+    asm volatile(
+        "rdsvl  %[res], #1\n"
+        "lsr    %[res], %[res], #2\n"
+        : [res] "=r" (cnt) ::
+    );
+    return cnt;
+}
+
+#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16
+// Outer product kernel.
+// Computes a 2SVL x 2SVL block of C, utilizing all four FP32 tiles of ZA.
+__attribute__((always_inline)) inline void
+kernel_2x2(const float *A, float *B, float *C, size_t shared_dim,
+           size_t ldc, size_t block_rows, size_t block_cols, float alpha, 
+           float beta, uint64_t row_idx, uint64_t col_idx)
+    __arm_out("za") __arm_streaming {
+
+  const uint64_t svl = svcntw();
+  size_t ldb = ldc;
+  // Predicate set-up
+  svbool_t pg = svptrue_b32();
+  svbool_t pg_a_0 = svwhilelt_b32_u64(0, block_rows);
+  svbool_t pg_a_1 = svwhilelt_b32_u64(svl, block_rows);
+
+  svbool_t pg_b_0 = svwhilelt_b32_u64(0, block_cols);
+  svbool_t pg_b_1 = svwhilelt_b32_u64(svl, block_cols);
+
+#define pg_c_0 pg_b_0
+#define pg_c_1 pg_b_1
+
+  svzero_za();
+  svfloat32_t beta_vec = svdup_f32(beta);
+
+  // Load C to ZA
+  for (size_t i = 0; i < MIN(svl, block_rows); i++) {
+    svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]);
+    row_c_0 = svmul_x(pg, beta_vec, row_c_0);
+    svwrite_hor_za32_f32_m(/*tile*/0, /*slice*/i, pg_c_0, row_c_0);
+    
+    svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]);
+    row_c_1 = svmul_x(pg, beta_vec, row_c_1);
+    svwrite_hor_za32_f32_m(/*tile*/1, /*slice*/i, pg_c_1, row_c_1);
+  }
+  for (size_t i = svl; i < block_rows; i++) {
+    svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]);
+    row_c_0 = svmul_x(pg, beta_vec, row_c_0);
+    svwrite_hor_za32_f32_m(/*tile*/2, /*slice*/i, pg_c_0, row_c_0);
+
+    svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]);
+    row_c_1 = svmul_x(pg, beta_vec, row_c_1);
+    svwrite_hor_za32_f32_m(/*tile*/3, /*slice*/i, pg_c_1, row_c_1);
+  }
+  
+  svfloat32_t alpha_vec = svdup_f32(alpha);
+  // Iterate through shared dimension (K)
+  for (size_t k = 0; k < shared_dim; k++) {
+#if !defined(TRANSA)
+    // Load column of A
+    svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * svl]);
+    col_a_0 = svmul_x(pg, alpha_vec, col_a_0);
+    svfloat32_t col_a_1 = svld1(pg_a_1, &A[(k + shared_dim) * svl]);
+    col_a_1 = svmul_x(pg, alpha_vec, col_a_1);
+    
+    // Load row of A**T
+    svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * svl]);
+    svfloat32_t row_b_1 = svld1(pg_b_1, &B[(k + shared_dim) * svl]);
+#else
+    // Load column of A**T
+    svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * ldb]);
+    col_a_0 = svmul_x(pg, alpha_vec, col_a_0);
+
+    svfloat32_t col_a_1 = svld1(pg_a_1, &A[k * ldb + svl]);
+    col_a_1 = svmul_x(pg, alpha_vec, col_a_1);
+    
+    // Load row of A
+    svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * ldb]);
+    svfloat32_t row_b_1 = svld1(pg_b_1, &B[k * ldb + svl]);    
+#endif
+    // Perform outer product
+    svmopa_za32_m(/*tile*/0, pg, pg, col_a_0, row_b_0);
+    svmopa_za32_m(/*tile*/1, pg, pg, col_a_0, row_b_1);
+    svmopa_za32_m(/*tile*/2, pg, pg, col_a_1, row_b_0);
+    svmopa_za32_m(/*tile*/3, pg, pg, col_a_1, row_b_1);
+  }
+
+#if defined(UPPER)
+#define pg_c_0_full pg_c_0
+#define pg_c_1_full pg_c_1
+  
+  bool need_update_pg_b = true;
+  size_t last_invalid_index = col_idx - row_idx;
+  // For Upper, If col_idx - row_idx >= 2*svl, we don't need to update the predicate due to all elements above the digonal
+  if (col_idx - row_idx >= 2*svl) {
+      need_update_pg_b = false;
+  }
+  // Store to C from ZA
+  for (size_t i = 0; i < MIN(svl, block_rows); i++, last_invalid_index++) {
+    if (need_update_pg_b) {
+       pg_c_0 = svnot_b_z(pg_c_0_full, svwhilelt_b32_u64(0, last_invalid_index));
+       pg_c_1 = svnot_b_z(pg_c_1_full, svwhilelt_b32_u64(svl, last_invalid_index));
+    }
+    
+    svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]);
+    svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }
+  for (size_t i = svl; i < block_rows; i++,last_invalid_index++) {
+      if (need_update_pg_b) {
+        pg_c_0 = svnot_b_z(pg_c_0_full, svwhilelt_b32_u64(0, last_invalid_index));
+        pg_c_1 = svnot_b_z(pg_c_1_full, svwhilelt_b32_u64(svl, last_invalid_index));
+      }
+      svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]);
+      svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }
+#else
+  // Store to C from ZA
+  size_t valid_index = row_idx - col_idx + 1;
+  for (size_t i = 0; i < MIN(svl, block_rows); i++, valid_index++) {
+    pg_c_0 = svwhilelt_b32_u64(0, MIN(valid_index, block_cols));
+    pg_c_1 = svwhilelt_b32_u64(svl, MIN(valid_index, block_cols));
+    svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]);
+    svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }
+  for (size_t i = svl; i < block_rows; i++, valid_index++) {
+    pg_c_0 = svwhilelt_b32_u64(0, MIN(valid_index, block_cols));
+    pg_c_1 = svwhilelt_b32_u64(svl, MIN(valid_index, block_cols));
+    svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]);
+    svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }  
+#endif 
+}
+
+__arm_new("za") __arm_locally_streaming
+static void ssyrk_direct_sme1_2VLx2VL(uint64_t n, uint64_t k, const float* alpha,\
+                                   const float *ba, const float* beta, float *restrict bc) {
+  const uint64_t num_rows = n;
+  const uint64_t num_cols = n;
+
+  const float *restrict a_ptr = ba;
+  const float *restrict b_ptr = ba;
+  float *restrict c_ptr = bc;
+
+  const uint64_t svl = svcntw();
+  const uint64_t ldc = n;
+
+  // Block over rows of C (panels of A)
+  uint64_t row_idx = 0;
+
+  // 2x2 loop
+  uint64_t row_batch = 2*svl;
+
+  // Block over row dimension of C
+  for (; row_idx < num_rows; row_idx += row_batch) {
+    row_batch = MIN(row_batch, num_rows - row_idx);
+    uint64_t col_batch = 2*svl;
+#if defined(UPPER)
+    // for UPLO is upper, Start from column col_idx = rows_index to ensure we only process the upper triangle (col_idx >= rows_index)
+    for (uint64_t col_idx = row_idx; col_idx < num_cols; col_idx += col_batch) {
+       col_batch = MIN(col_batch, num_cols - col_idx);
+#else
+    // for UPLO is lower, we only process the lower triangle part (col_idx <= row_idxx)
+    for (uint64_t col_idx = 0; col_idx < num_cols && col_idx <= row_idx; col_idx += col_batch) {
+#endif
+      col_batch = MIN(col_batch, num_cols - col_idx);
+#if !defined(TRANSA)
+      kernel_2x2(&a_ptr[row_idx * k], &b_ptr[col_idx * k],
+                 &c_ptr[row_idx * ldc + col_idx], k,
+				 ldc, row_batch, col_batch, *alpha, *beta, row_idx, col_idx);
+#else
+      kernel_2x2(&a_ptr[row_idx], &b_ptr[col_idx],
+                &c_ptr[row_idx * ldc + col_idx], k,
+                ldc, row_batch, col_batch, *alpha, *beta, row_idx, col_idx);
+#endif
+
+    }
+  }
+  return;
+}
+
+#else
+static void ssyrk_direct_sme1_2VLx2VL(uint64_t n, uint64_t k, const float* alpha,\
+                                   const float *ba, const float* beta, float *restrict bc){}
+#endif
+
+void CNAME (BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\
+            BLASLONG strideA, float beta, float * __restrict C, BLASLONG strideC){
+#if !defined(TRANSA)          
+        uint64_t n_mod, vl_elms;
+        
+        vl_elms = sve_cntw();
+
+        n_mod = ceil((double)N/(double)vl_elms) * vl_elms;
+
+        float *A_mod = (float *) malloc(n_mod*K*sizeof(float));
+	    
+	    /* Prevent compiler optimization by reading from memory instead
+	     * of reading directly from vector (z) registers.
+	     * */
+        asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7",
+                         "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15",
+                         "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7",
+                         "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15",
+                         "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23",
+                         "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31");
+      
+        /* Pre-process the left matrix to make it suitable for 
+           matrix sum of outer-product calculation
+         */
+        sgemm_direct_sme1_preprocess(N, K, A, A_mod);
+        asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7",
+                         "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15",
+                         "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7",
+                         "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15",
+                         "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23",
+                         "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31");
+        ssyrk_direct_sme1_2VLx2VL(N, K, &alpha, A_mod, &beta, C);
+        free(A_mod);
+#else
+        ssyrk_direct_sme1_2VLx2VL(N, K, &alpha, A, &beta, C);
+#endif
+        
+}
+
+#else
+
+void CNAME (BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\
+            BLASLONG strideA, float beta, float * __restrict C, BLASLONG strideC){}
+ 
+#endif

kernel/arm64/strmm_direct_arm64_sme1.c

@@ -0,0 +1,260 @@
+/*
+ Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ SPDX-License-Identifier: BSD-3-Clause-Clear
+*/
+
+#include "common.h"
+#include <stdlib.h>
+#include <inttypes.h>
+#include <math.h>
+//#include "sme_abi.h"
+#if defined(HAVE_SME)
+
+#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16
+#include <arm_sme.h>
+#endif
+
+/* Function Definitions */
+static uint64_t sve_cntw() {
+    uint64_t cnt;
+    asm volatile(
+        "rdsvl  %[res], #1\n"
+        "lsr    %[res], %[res], #2\n"
+        : [res] "=r" (cnt) ::
+    );
+    return cnt;
+}
+
+#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16
+
+// Transpose 1SVL x N panel of A
+__attribute__((always_inline))
+inline static void transpose_panel_lower(const float *restrict a, float *restrict b, 
+                uint64_t rows, uint64_t cols,
+                uint64_t a_step, uint64_t rows_index)
+__arm_out("za") __arm_streaming {
+    // for Lower Trangular Matrix
+    uint64_t svl = svcntw();
+    uint64_t col_batch = svl;
+
+    svzero_za();
+    uint64_t last_rows_index = rows_index + rows - 1;
+    for (uint64_t k = 0; k < cols; k += col_batch) {
+        if (last_rows_index < k) {
+            // Early exit: if all rows are above the diagonal, no valid elements remain
+            break;
+        }
+        // Load to horizontal slices
+        for (uint64_t row = 0; row < rows; row++) {
+            svbool_t pg_row = svwhilelt_b32_u64(k, MIN(rows_index + row + 1, cols));
+            svld1_hor_za32(0, row, pg_row, &a[row * a_step + k]);
+        }
+
+        // Save from vertical slices
+        col_batch = MIN(col_batch, cols - k);
+        for (uint64_t col = 0; col < col_batch; col++) {
+            svst1_ver_za32(0, col, svptrue_b32(), &b[(col + k) * svl]);
+        }
+    }
+}
+
+__attribute__((always_inline))
+inline static void transpose_panel_upper(const float *restrict a, float *restrict b, 
+                uint64_t rows, uint64_t cols,
+                uint64_t a_step, uint64_t rows_index)
+__arm_out("za") __arm_streaming {
+    // for Upper Trangular Matrix
+    uint64_t svl = svcntw();
+    uint64_t col_batch = svl;
+
+    svzero_za();
+    // Start from column k = rows_index to ensure we only process the upper triangle (k >= rows_index)
+    for (uint64_t k = rows_index; k < cols; k += col_batch) {
+        // Load to horizontal slices
+        for (uint64_t row = 0; row < rows; row++) {
+            svbool_t pg_row = svwhilelt_b32_u64(k, cols);
+            svld1_hor_za32(0, row, pg_row, &a[row * a_step + k]);
+        }
+
+        // Save from vertical slices
+        col_batch = MIN(col_batch, cols - k);
+        for (uint64_t col = 0, real_col = k; col < col_batch; col++, real_col++) {
+            // Only the upper triangular part of the matrix is stored.
+            svbool_t pg_col = svwhilelt_b32_u64(rows_index, real_col + 1);
+            svst1_ver_za32(0, col, pg_col, &b[(col + k) * svl]);  
+        }
+    }
+}
+
+__arm_new("za") __arm_locally_streaming
+static void strmm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,
+                const float *restrict a, float *restrict a_mod) {
+    const uint64_t num_rows = nbr;
+    uint64_t row_batch = svcntsw();
+    for (uint64_t row_idx = 0; row_idx < num_rows; row_idx += row_batch) {
+        // Transpose 1SVL x N panel of A
+        row_batch = MIN(row_batch, num_rows - row_idx);
+#if !defined(UPPER)
+        transpose_panel_lower(&a[row_idx * nbc], &a_mod[row_idx * nbc], row_batch, nbc, nbc, row_idx);
+#else
+        transpose_panel_upper(&a[row_idx * nbc], &a_mod[row_idx * nbc], row_batch, nbc, nbc, row_idx);
+#endif
+    }
+}
+
+// Outer product kernel.
+// Computes a 2SVL x 2SVL block of C, utilizing all four FP32 tiles of ZA.
+__attribute__((always_inline)) inline void
+kernel_2x2(const float *A, const float *B, float *C, size_t shared_dim,
+           size_t ldc, size_t block_rows, size_t block_cols, float alpha, uint64_t row_idx)
+    __arm_out("za") __arm_streaming {
+    const uint64_t svl = svcntw();
+    size_t ldb = ldc;
+    // Predicate set-up
+    svbool_t pg = svptrue_b32();    
+    svbool_t pg_a_0 = svwhilelt_b32_u64(0, block_rows);
+    svbool_t pg_a_1 = svwhilelt_b32_u64(svl, block_rows);
+
+#if (!defined(TRANSA) && !defined(UPPER)) || (defined(TRANSA) && defined(UPPER))
+#define pg_a_0_full pg_a_0
+#define pg_a_1_full pg_a_1
+#endif
+    svbool_t pg_b_0 = svwhilelt_b32_u64(0, block_cols);
+    svbool_t pg_b_1 = svwhilelt_b32_u64(svl, block_cols);
+
+#define pg_c_0 pg_b_0
+#define pg_c_1 pg_b_1
+
+    svzero_za();
+    svfloat32_t alpha_vec = svdup_f32(alpha);
+    // Iterate through shared dimension (K)
+#if (!defined(TRANSA) && defined(UPPER)) || (defined(TRANSA) && !defined(UPPER))
+    for (size_t k = row_idx, valid_index = 1; k < shared_dim; k++,valid_index++) {
+        pg_a_0 = svwhilelt_b32_u64(0, MIN(valid_index, block_rows));
+        pg_a_1 = svwhilelt_b32_u64(svl, MIN(valid_index, block_rows));
+#else
+    for (size_t k = 0; k < MIN(row_idx + block_rows, shared_dim); k++) {
+        // If k exceeds row_idx, mask out rows before (k - row_idx)
+        // This ensures only valid rows are included for lower triangular logic.
+        if (k > row_idx) {
+            pg_a_0 = svnot_b_z(pg_a_0_full, svwhilelt_b32_u64(0, k - row_idx));  
+            pg_a_1 = svnot_b_z(pg_a_1_full, svwhilelt_b32_u64(svl, k - row_idx));
+        }
+#endif
+
+#if !defined(TRANSA)
+        // Load column of A
+        svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * svl]);
+        svfloat32_t col_a_1 = svld1(pg_a_1, &A[(k + shared_dim) * svl]);
+#else   
+        svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * shared_dim]);
+        svfloat32_t col_a_1 = svld1(pg_a_1, &A[k * shared_dim + svl]);
+#endif       
+        col_a_0 = svmul_x(pg_a_0, alpha_vec, col_a_0);
+        col_a_1 = svmul_x(pg_a_1, alpha_vec, col_a_1);
+        // Load row of B
+        svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * ldb]);
+        svfloat32_t row_b_1 = svld1(pg_b_1, &B[k * ldb + svl]);
+        // Perform outer product
+        svmopa_za32_m(/*tile*/0, pg_a_0, pg, col_a_0, row_b_0);
+        svmopa_za32_m(/*tile*/1, pg_a_0, pg, col_a_0, row_b_1);
+        svmopa_za32_m(/*tile*/2, pg_a_1, pg, col_a_1, row_b_0);
+        svmopa_za32_m(/*tile*/3, pg_a_1, pg, col_a_1, row_b_1);
+    }
+
+    // Store to C from ZA
+    for (size_t i = 0; i < MIN(svl, block_rows); i++) {
+      svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]);
+      svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+    }
+    for (size_t i = svl; i < block_rows; i++) {
+      svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]);
+      svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+    }
+
+}
+
+__arm_new("za") __arm_locally_streaming
+static inline void strmm_direct_alpha_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\
+                                   const float *ba, float *restrict bb) {
+    const uint64_t num_rows = m;
+    const uint64_t num_cols = n;
+    
+    const float *restrict a_ptr = ba;
+    const float *restrict b_ptr = bb;
+    float *restrict c_ptr = bb;
+    
+    const uint64_t svl = svcntw();
+    const uint64_t svl_x2 = 2*svl;
+    const uint64_t ldc = n; 
+    
+
+    uint64_t row_idx = 0;
+#if (!defined(TRANSA) && defined(UPPER)) || (defined(TRANSA) && !defined(UPPER))
+    // 2x2 loop
+    uint64_t row_batch = svl_x2;
+    // Block over rows of C (panels of A)
+    for (; row_idx < num_rows; row_idx += row_batch) {
+        row_batch = MIN(row_batch, num_rows - row_idx);
+#else
+    // Calculate the remainder of num_rows divided by 2VL to determine tail tile size
+    uint64_t row_batch = num_rows % svl_x2;
+    // If there's no remainder, use full tile size (2VL) for initial batch
+    if (row_batch == 0) row_batch = svl_x2;
+    // Loop from bottom to top, processing rows in batches
+    for (uint64_t index = num_rows; index > 0; index -= row_batch, row_batch = svl_x2) {
+        // Compute the starting row index for the current batch
+        row_idx = index - row_batch;        
+#endif
+        uint64_t col_idx = 0;
+        uint64_t col_batch = svl_x2;
+        // Block over column dimension of C
+        for (; col_idx < num_cols; col_idx += col_batch) {
+            col_batch = MIN(col_batch, num_cols - col_idx);
+#if !defined(TRANSA)
+            kernel_2x2(&a_ptr[row_idx * k], &b_ptr[col_idx],
+                    &c_ptr[row_idx * ldc + col_idx], k,
+                    ldc, row_batch, col_batch, *alpha, row_idx);
+#else
+            kernel_2x2(&a_ptr[row_idx], &b_ptr[col_idx],
+                &c_ptr[row_idx * ldc + col_idx], k,
+                ldc, row_batch, col_batch, *alpha, row_idx);
+#endif
+        }
+    }
+
+    return;
+}
+
+#else
+static void strmm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,
+                                         const float *restrict a, float *restrict a_mod) {}
+static void strmm_direct_alpha_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\
+                                            const float *ba, float *restrict bb){}
+#endif
+
+void CNAME (BLASLONG M, BLASLONG N, float alpha, float * __restrict A,\
+            BLASLONG strideA, float * __restrict B, BLASLONG strideB){
+#if !defined(TRANSA)
+    uint64_t m_mod, vl_elms;
+    
+    vl_elms = sve_cntw();
+
+    m_mod = ceil((double)M/(double)vl_elms) * vl_elms;
+
+    float *A_mod = (float *) malloc(m_mod*M*sizeof(float));
+    strmm_direct_sme1_preprocess(M, M, A, A_mod);
+    /* Calculate B = alpha*A*B*/
+    strmm_direct_alpha_sme1_2VLx2VL(M, M, N, &alpha, A_mod, B);
+    free(A_mod);
+#else
+    strmm_direct_alpha_sme1_2VLx2VL(M, M, N, &alpha, A, B);
+#endif
+}
+
+#else
+void CNAME (BLASLONG M, BLASLONG N, float alpha, float * __restrict A,\
+            BLASLONG strideA, float * __restrict B, BLASLONG strideB){
+            }
+
+#endif

kernel/generic/conversion_macros.h

@@ -27,6 +27,7 @@
  * *****************************************************************************/
 
 #if defined(BFLOAT16) && defined(BFLOAT16CONVERSION)
+
 static float
 bfloat16tof32 (bfloat16 value)
 {
@@ -48,17 +49,34 @@ static bfloat16 f32tobfloat16(float value) {
 #ifdef BGEMM
 #define ALPHA bfloat16tof32(alpha)
 #define BETA bfloat16tof32(beta)
-#define BF16TOF32(x) (bfloat16tof32(x))
-#define F32TOBF16(x) (f32tobfloat16(x))
+#define TO_F32(x) (bfloat16tof32(x))
+#define TO_OUTPUT(x) (f32tobfloat16(x))
 #else
 #define ALPHA alpha
 #define BETA beta
-#define BF16TOF32(x) (bfloat16tof32(x))
-#define F32TOBF16(x) x
+#define TO_F32(x) (bfloat16tof32(x))
+#define TO_OUTPUT(x) x
 #endif
+
+#elif defined(HFLOAT16)
+
+#ifdef HGEMM
+#define ALPHA (float)(alpha)
+#define BETA (float)(beta)
+#define TO_F32(x) ((float)(x))
+#define TO_OUTPUT(x) ((_Float16)(x))
 #else
 #define ALPHA alpha
 #define BETA beta
-#define BF16TOF32(x) x
-#define F32TOBF16(x) x
+#define TO_F32(x) ((float)(x))
+#define TO_OUTPUT(x) x
+#endif
+
+#else
+
+#define ALPHA alpha
+#define BETA beta
+#define TO_F32(x) x
+#define TO_OUTPUT(x) x
+
 #endif

kernel/generic/gemmkernel_2x2.c

@@ -27,7 +27,8 @@
  * *****************************************************************************/
 
 #include "common.h"
-#include "bf16_macros.h"
+
+#include "conversion_macros.h"
 
 int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,FLOAT* C,BLASLONG ldc
 #ifdef TRMMKERNEL
@@ -60,36 +61,36 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                {
                   load0 = ptrba[2*0+0];
                   load1 = ptrbb[2*0+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   load2 = ptrba[2*0+1];
-                  res1 = res1+BF16TOF32(load2)*BF16TOF32(load1);
+                  res1 = res1+TO_F32(load2)*TO_F32(load1);
                   load3 = ptrbb[2*0+1];
-                  res2 = res2+BF16TOF32(load0)*BF16TOF32(load3);
-                  res3 = res3+BF16TOF32(load2)*BF16TOF32(load3);
+                  res2 = res2+TO_F32(load0)*TO_F32(load3);
+                  res3 = res3+TO_F32(load2)*TO_F32(load3);
                   load4 = ptrba[2*1+0];
                   load5 = ptrbb[2*1+0];
-                  res0 = res0+BF16TOF32(load4)*BF16TOF32(load5);
+                  res0 = res0+TO_F32(load4)*TO_F32(load5);
                   load6 = ptrba[2*1+1];
-                  res1 = res1+BF16TOF32(load6)*BF16TOF32(load5);
+                  res1 = res1+TO_F32(load6)*TO_F32(load5);
                   load7 = ptrbb[2*1+1];
-                  res2 = res2+BF16TOF32(load4)*BF16TOF32(load7);
-                  res3 = res3+BF16TOF32(load6)*BF16TOF32(load7);
+                  res2 = res2+TO_F32(load4)*TO_F32(load7);
+                  res3 = res3+TO_F32(load6)*TO_F32(load7);
                   load0 = ptrba[2*2+0];
                   load1 = ptrbb[2*2+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   load2 = ptrba[2*2+1];
-                  res1 = res1+BF16TOF32(load2)*BF16TOF32(load1);
+                  res1 = res1+TO_F32(load2)*TO_F32(load1);
                   load3 = ptrbb[2*2+1];
-                  res2 = res2+BF16TOF32(load0)*BF16TOF32(load3);
-                  res3 = res3+BF16TOF32(load2)*BF16TOF32(load3);
+                  res2 = res2+TO_F32(load0)*TO_F32(load3);
+                  res3 = res3+TO_F32(load2)*TO_F32(load3);
                   load4 = ptrba[2*3+0];
                   load5 = ptrbb[2*3+0];
-                  res0 = res0+BF16TOF32(load4)*BF16TOF32(load5);
+                  res0 = res0+TO_F32(load4)*TO_F32(load5);
                   load6 = ptrba[2*3+1];
-                  res1 = res1+BF16TOF32(load6)*BF16TOF32(load5);
+                  res1 = res1+TO_F32(load6)*TO_F32(load5);
                   load7 = ptrbb[2*3+1];
-                  res2 = res2+BF16TOF32(load4)*BF16TOF32(load7);
-                  res3 = res3+BF16TOF32(load6)*BF16TOF32(load7);
+                  res2 = res2+TO_F32(load4)*TO_F32(load7);
+                  res3 = res3+TO_F32(load6)*TO_F32(load7);
                   ptrba = ptrba+8;
                   ptrbb = ptrbb+8;
                }
@@ -97,23 +98,23 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                {
                   load0 = ptrba[2*0+0];
                   load1 = ptrbb[2*0+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   load2 = ptrba[2*0+1];
-                  res1 = res1+BF16TOF32(load2)*BF16TOF32(load1);
+                  res1 = res1+TO_F32(load2)*TO_F32(load1);
                   load3 = ptrbb[2*0+1];
-                  res2 = res2+BF16TOF32(load0)*BF16TOF32(load3);
-                  res3 = res3+BF16TOF32(load2)*BF16TOF32(load3);
+                  res2 = res2+TO_F32(load0)*TO_F32(load3);
+                  res3 = res3+TO_F32(load2)*TO_F32(load3);
                   ptrba = ptrba+2;
                   ptrbb = ptrbb+2;
                }
              res0 = res0*ALPHA;
-             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             C0[0] = TO_OUTPUT(TO_F32(C0[0])+res0);
              res1 = res1*ALPHA;
-             C0[1] = F32TOBF16(BF16TOF32(C0[1])+res1);
+             C0[1] = TO_OUTPUT(TO_F32(C0[1])+res1);
              res2 = res2*ALPHA;
-             C1[0] = F32TOBF16(BF16TOF32(C1[0])+res2);
+             C1[0] = TO_OUTPUT(TO_F32(C1[0])+res2);
              res3 = res3*ALPHA;
-             C1[1] = F32TOBF16(BF16TOF32(C1[1])+res3);
+             C1[1] = TO_OUTPUT(TO_F32(C1[1])+res3);
              C0 = C0+2;
              C1 = C1+2;
           }
@@ -126,16 +127,16 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                {
                   load0 = ptrba[0+0];
                   load1 = ptrbb[2*0+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   load2 = ptrbb[2*0+1];
-                  res1 = res1+BF16TOF32(load0)*BF16TOF32(load2);
+                  res1 = res1+TO_F32(load0)*TO_F32(load2);
                   ptrba = ptrba+1;
                   ptrbb = ptrbb+2;
                }
              res0 = res0*ALPHA;
-             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             C0[0] = TO_OUTPUT(TO_F32(C0[0])+res0);
              res1 = res1*ALPHA;
-             C1[0] = F32TOBF16(BF16TOF32(C1[0])+res1);
+             C1[0] = TO_OUTPUT(TO_F32(C1[0])+res1);
              C0 = C0+1;
              C1 = C1+1;
           }
@@ -157,16 +158,16 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                {
                   load0 = ptrba[2*0+0];
                   load1 = ptrbb[0+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   load2 = ptrba[2*0+1];
-                  res1 = res1+BF16TOF32(load2)*BF16TOF32(load1);
+                  res1 = res1+TO_F32(load2)*TO_F32(load1);
                   ptrba = ptrba+2;
                   ptrbb = ptrbb+1;
                }
              res0 = res0*ALPHA;
-             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             C0[0] = TO_OUTPUT(TO_F32(C0[0])+res0);
              res1 = res1*ALPHA;
-             C0[1] = F32TOBF16(BF16TOF32(C0[1])+res1);
+             C0[1] = TO_OUTPUT(TO_F32(C0[1])+res1);
              C0 = C0+2;
           }
         for (i=0; i<(bm&1); i+=1)
@@ -177,12 +178,12 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                {
                   load0 = ptrba[0+0];
                   load1 = ptrbb[0+0];
-                  res0 = res0+BF16TOF32(load0)*BF16TOF32(load1);
+                  res0 = res0+TO_F32(load0)*TO_F32(load1);
                   ptrba = ptrba+1;
                   ptrbb = ptrbb+1;
                }
              res0 = res0*ALPHA;
-             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             C0[0] = TO_OUTPUT(TO_F32(C0[0])+res0);
              C0 = C0+1;
           }
         k = (bk<<0);

kernel/generic/gemv_n.c

@@ -26,15 +26,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *****************************************************************************/
 
 #include "common.h"
-#include "bf16_macros.h"
+
+#include "conversion_macros.h"
 
 int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y)
 {
-    BLASLONG i;
     BLASLONG ix, iy;
-    BLASLONG j;
-    FLOAT *a_ptr;
-#ifdef BGEMM
+    IFLOAT *a_ptr;
+#if defined(BGEMM) || defined(HGEMM)
     float temp;
 #else
     FLOAT temp;
@@ -49,18 +48,18 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *
         a_ptr = a;
         for (BLASLONG j = 0; j < n; j++)
         {
-            temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]);
+            temp += TO_F32(a_ptr[i]) * TO_F32(x[ix]);
             ix += inc_x;
             a_ptr += lda;
         }
 
         if (BETA == ZERO)
         {
-            y[iy] = F32TOBF16(ALPHA * temp);
+            y[iy] = TO_OUTPUT(ALPHA * temp);
         }
         else
         {
-            y[iy] = F32TOBF16(ALPHA * temp + BETA * BF16TOF32(y[iy]));
+            y[iy] = TO_OUTPUT(ALPHA * temp + BETA * TO_F32(y[iy]));
         }
 
         iy += inc_y;

kernel/generic/gemv_t.c

@@ -26,15 +26,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *****************************************************************************/
 
 #include "common.h"
-#include "bf16_macros.h"
+
+#include "conversion_macros.h"
 
 int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y)
 {
     BLASLONG i;
     BLASLONG ix, iy;
     BLASLONG j;
-    FLOAT *a_ptr;
-#ifdef BGEMM
+    IFLOAT *a_ptr;
+#if defined(BGEMM) || defined(HGEMM)
     float temp;
 #else
     FLOAT temp;
@@ -49,10 +50,17 @@ int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *
         ix = 0;
         for (i = 0; i < m; i++)
         {
-            temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]);
+            temp += TO_F32(a_ptr[i]) * TO_F32(x[ix]);
             ix += inc_x;
         }
-        y[iy] += F32TOBF16(ALPHA * temp);
+        if (BETA == ZERO)
+        {
+            y[iy] = TO_OUTPUT(ALPHA * temp);
+        }
+        else
+        {
+            y[iy] = TO_OUTPUT(ALPHA * temp + BETA * TO_F32(y[iy]));
+        }
         iy += inc_y;
         a_ptr += lda;
     }

kernel/loongarch64/cgemm_kernel_2x2_lasx.S

@@ -212,15 +212,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
     SDARG      $r27,   $sp,   32
-    ST         $f23,   $sp,   40
-    ST         $f24,   $sp,   48
-    ST         $f25,   $sp,   56
-    ST         $f26,   $sp,   64
-    ST         $f27,   $sp,   72
-    ST         $f28,   $sp,   80
-    ST         $f29,   $sp,   88
-    ST         $f30,   $sp,   96
-    ST         $f31,   $sp,   104
+    fst.d      $f23,   $sp,   40
+    fst.d      $f24,   $sp,   48
+    fst.d      $f25,   $sp,   56
+    fst.d      $f26,   $sp,   64
+    fst.d      $f27,   $sp,   72
+    fst.d      $f28,   $sp,   80
+    fst.d      $f29,   $sp,   88
+    fst.d      $f30,   $sp,   96
+    fst.d      $f31,   $sp,   104
     ST         ALPHA_R,$sp,   112
     ST         ALPHA_I,$sp,   120
 
@@ -841,15 +841,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
     LDARG      $r27,   $sp,   32
-    LD         $f23,   $sp,   40
-    LD         $f24,   $sp,   48
-    LD         $f25,   $sp,   56
-    LD         $f26,   $sp,   64
-    LD         $f27,   $sp,   72
-    LD         $f28,   $sp,   80
-    LD         $f29,   $sp,   88
-    LD         $f30,   $sp,   96
-    LD         $f31,   $sp,   104
+    fld.d      $f23,   $sp,   40
+    fld.d      $f24,   $sp,   48
+    fld.d      $f25,   $sp,   56
+    fld.d      $f26,   $sp,   64
+    fld.d      $f27,   $sp,   72
+    fld.d      $f28,   $sp,   80
+    fld.d      $f29,   $sp,   88
+    fld.d      $f30,   $sp,   96
+    fld.d      $f31,   $sp,   104
 
     addi.d     $sp,    $sp,   128
     jirl       $r0,    $r1,   0x0

kernel/loongarch64/cgemm_kernel_2x2_lsx.S

@@ -172,15 +172,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
     SDARG      $r27,   $sp,   32
-    ST         $f23,   $sp,   40
-    ST         $f24,   $sp,   48
-    ST         $f25,   $sp,   56
-    ST         $f26,   $sp,   64
-    ST         $f27,   $sp,   72
-    ST         $f28,   $sp,   80
-    ST         $f29,   $sp,   88
-    ST         $f30,   $sp,   96
-    ST         $f31,   $sp,   104
+    fst.d      $f23,   $sp,   40
+    fst.d      $f24,   $sp,   48
+    fst.d      $f25,   $sp,   56
+    fst.d      $f26,   $sp,   64
+    fst.d      $f27,   $sp,   72
+    fst.d      $f28,   $sp,   80
+    fst.d      $f29,   $sp,   88
+    fst.d      $f30,   $sp,   96
+    fst.d      $f31,   $sp,   104
     ST         ALPHA_R,$sp,   112
     ST         ALPHA_I,$sp,   120
 
@@ -796,15 +796,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
     LDARG      $r27,   $sp,   32
-    LD         $f23,   $sp,   40
-    LD         $f24,   $sp,   48
-    LD         $f25,   $sp,   56
-    LD         $f26,   $sp,   64
-    LD         $f27,   $sp,   72
-    LD         $f28,   $sp,   80
-    LD         $f29,   $sp,   88
-    LD         $f30,   $sp,   96
-    LD         $f31,   $sp,   104
+    fld.d      $f23,   $sp,   40
+    fld.d      $f24,   $sp,   48
+    fld.d      $f25,   $sp,   56
+    fld.d      $f26,   $sp,   64
+    fld.d      $f27,   $sp,   72
+    fld.d      $f28,   $sp,   80
+    fld.d      $f29,   $sp,   88
+    fld.d      $f30,   $sp,   96
+    fld.d      $f31,   $sp,   104
 
     addi.d     $sp,    $sp,   128
     jirl       $r0,    $r1,   0x0

kernel/loongarch64/cgemm_kernel_8x4_lsx.S

@@ -176,15 +176,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
     SDARG      $r27,   $sp,   32
-    ST         $f23,   $sp,   40
-    ST         $f24,   $sp,   48
-    ST         $f25,   $sp,   56
-    ST         $f26,   $sp,   64
-    ST         $f27,   $sp,   72
-    ST         $f28,   $sp,   80
-    ST         $f29,   $sp,   88
-    ST         $f30,   $sp,   96
-    ST         $f31,   $sp,   104
+    fst.d      $f23,   $sp,   40
+    fst.d      $f24,   $sp,   48
+    fst.d      $f25,   $sp,   56
+    fst.d      $f26,   $sp,   64
+    fst.d      $f27,   $sp,   72
+    fst.d      $f28,   $sp,   80
+    fst.d      $f29,   $sp,   88
+    fst.d      $f30,   $sp,   96
+    fst.d      $f31,   $sp,   104
     ST         ALPHA_R,$sp,   112
     ST         ALPHA_I,$sp,   120
 
@@ -3297,15 +3297,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
     LDARG      $r27,   $sp,   32
-    LD         $f23,   $sp,   40
-    LD         $f24,   $sp,   48
-    LD         $f25,   $sp,   56
-    LD         $f26,   $sp,   64
-    LD         $f27,   $sp,   72
-    LD         $f28,   $sp,   80
-    LD         $f29,   $sp,   88
-    LD         $f30,   $sp,   96
-    LD         $f31,   $sp,   104
+    fld.d      $f23,   $sp,   40
+    fld.d      $f24,   $sp,   48
+    fld.d      $f25,   $sp,   56
+    fld.d      $f26,   $sp,   64
+    fld.d      $f27,   $sp,   72
+    fld.d      $f28,   $sp,   80
+    fld.d      $f29,   $sp,   88
+    fld.d      $f30,   $sp,   96
+    fld.d      $f31,   $sp,   104
 
     addi.d     $sp,    $sp,   128
     jirl       $r0,    $r1,   0x0

kernel/loongarch64/dgemm_kernel_8x4.S

@@ -893,14 +893,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG    $r25,  $sp,   16
     SDARG    $r26,  $sp,   24
     SDARG    $r27,  $sp,   32
-    ST       $f24,  $sp,   40
-    ST       $f25,  $sp,   48
-    ST       $f26,  $sp,   56
-    ST       $f27,  $sp,   64
-    ST       $f28,  $sp,   72
-    ST       $f29,  $sp,   80
-    ST       $f30,  $sp,   88
-    ST       $f31,  $sp,   96
+    fst.d    $f24,  $sp,   40
+    fst.d    $f25,  $sp,   48
+    fst.d    $f26,  $sp,   56
+    fst.d    $f27,  $sp,   64
+    fst.d    $f28,  $sp,   72
+    fst.d    $f29,  $sp,   80
+    fst.d    $f30,  $sp,   88
+    fst.d    $f31,  $sp,   96
     ST       ALPHA, $sp,   104
 
 #if defined (TRMMKERNEL) && !defined(LEFT)
@@ -2879,14 +2879,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG    $r25,  $sp,   16
     LDARG    $r26,  $sp,   24
     LDARG    $r27,  $sp,   32
-    LD       $f24,  $sp,   40
-    LD       $f25,  $sp,   48
-    LD       $f26,  $sp,   56
-    LD       $f27,  $sp,   64
-    LD       $f28,  $sp,   72
-    LD       $f29,  $sp,   80
-    LD       $f30,  $sp,   88
-    LD       $f31,  $sp,   96
+    fld.d    $f24,  $sp,   40
+    fld.d    $f25,  $sp,   48
+    fld.d    $f26,  $sp,   56
+    fld.d    $f27,  $sp,   64
+    fld.d    $f28,  $sp,   72
+    fld.d    $f29,  $sp,   80
+    fld.d    $f30,  $sp,   88
+    fld.d    $f31,  $sp,   96
     addi.d   $sp,   $sp,   112
 
     jirl    $r0, $r1, 0x0

kernel/loongarch64/dgemm_ncopy_16.S

@@ -113,15 +113,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r29, $sp,  0x30
     SDARG      $r30, $sp,  0x38
     SDARG      $r31, $sp,  0x40
-    ST         $f23, $sp,  0x48
-    ST         $f24, $sp,  0x50
-    ST         $f25, $sp,  0x58
-    ST         $f26, $sp,  0x60
-    ST         $f27, $sp,  0x68
-    ST         $f28, $sp,  0x70
-    ST         $f29, $sp,  0x78
-    ST         $f30, $sp,  0x80
-    ST         $f31, $sp,  0x88
+    fst.d      $f23, $sp,  0x48
+    fst.d      $f24, $sp,  0x50
+    fst.d      $f25, $sp,  0x58
+    fst.d      $f26, $sp,  0x60
+    fst.d      $f27, $sp,  0x68
+    fst.d      $f28, $sp,  0x70
+    fst.d      $f29, $sp,  0x78
+    fst.d      $f30, $sp,  0x80
+    fst.d      $f31, $sp,  0x88
 
     move       TD,   DST
     move       TS,   SRC
@@ -678,15 +678,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r29, $sp,  0x30
     LDARG      $r30, $sp,  0x38
     LDARG      $r31, $sp,  0x40
-    LD         $f23, $sp,  0x48
-    LD         $f24, $sp,  0x50
-    LD         $f25, $sp,  0x58
-    LD         $f26, $sp,  0x60
-    LD         $f27, $sp,  0x68
-    LD         $f28, $sp,  0x70
-    LD         $f29, $sp,  0x78
-    LD         $f30, $sp,  0x80
-    LD         $f31, $sp,  0x88
+    fld.d      $f23, $sp,  0x48
+    fld.d      $f24, $sp,  0x50
+    fld.d      $f25, $sp,  0x58
+    fld.d      $f26, $sp,  0x60
+    fld.d      $f27, $sp,  0x68
+    fld.d      $f28, $sp,  0x70
+    fld.d      $f29, $sp,  0x78
+    fld.d      $f30, $sp,  0x80
+    fld.d      $f31, $sp,  0x88
     addi.d     $sp,  $sp,  0x90
     jirl       $r0,  $r1,  0x00
 

kernel/loongarch64/zgemm_kernel_2x2.S

@@ -123,13 +123,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r24,   $sp,   8
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
-    ST         $f23,   $sp,   32
-    ST         $f24,   $sp,   40
-    ST         $f25,   $sp,   48
-    ST         $f26,   $sp,   56
-    ST         $f27,   $sp,   64
-    ST         $f28,   $sp,   72
-    ST         $f29,   $sp,   80
+    fst.d      $f23,   $sp,   32
+    fst.d      $f24,   $sp,   40
+    fst.d      $f25,   $sp,   48
+    fst.d      $f26,   $sp,   56
+    fst.d      $f27,   $sp,   64
+    fst.d      $f28,   $sp,   72
+    fst.d      $f29,   $sp,   80
 
 
 #if defined (TRMMKERNEL) && !defined(LEFT)
@@ -834,13 +834,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r24,   $sp,   8
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
-    LD         $f23,   $sp,   32
-    LD         $f24,   $sp,   40
-    LD         $f25,   $sp,   48
-    LD         $f26,   $sp,   56
-    LD         $f27,   $sp,   64
-    LD         $f28,   $sp,   72
-    LD         $f29,   $sp,   80
+    fld.d      $f23,   $sp,   32
+    fld.d      $f24,   $sp,   40
+    fld.d      $f25,   $sp,   48
+    fld.d      $f26,   $sp,   56
+    fld.d      $f27,   $sp,   64
+    fld.d      $f28,   $sp,   72
+    fld.d      $f29,   $sp,   80
 
     addi.d     $sp,    $sp,   88
     jirl       $r0,    $r1,   0x0

kernel/loongarch64/zgemm_kernel_2x2_lasx.S

@@ -174,15 +174,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
     SDARG      $r27,   $sp,   32
-    ST         $f23,   $sp,   40
-    ST         $f24,   $sp,   48
-    ST         $f25,   $sp,   56
-    ST         $f26,   $sp,   64
-    ST         $f27,   $sp,   72
-    ST         $f28,   $sp,   80
-    ST         $f29,   $sp,   88
-    ST         $f30,   $sp,   96
-    ST         $f31,   $sp,   104
+    fst.d      $f23,   $sp,   40
+    fst.d      $f24,   $sp,   48
+    fst.d      $f25,   $sp,   56
+    fst.d      $f26,   $sp,   64
+    fst.d      $f27,   $sp,   72
+    fst.d      $f28,   $sp,   80
+    fst.d      $f29,   $sp,   88
+    fst.d      $f30,   $sp,   96
+    fst.d      $f31,   $sp,   104
     ST         ALPHA_R,$sp,   112
     ST         ALPHA_I,$sp,   120
 
@@ -806,15 +806,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
     LDARG      $r27,   $sp,   32
-    LD         $f23,   $sp,   40
-    LD         $f24,   $sp,   48
-    LD         $f25,   $sp,   56
-    LD         $f26,   $sp,   64
-    LD         $f27,   $sp,   72
-    LD         $f28,   $sp,   80
-    LD         $f29,   $sp,   88
-    LD         $f30,   $sp,   96
-    LD         $f31,   $sp,   104
+    fld.d      $f23,   $sp,   40
+    fld.d      $f24,   $sp,   48
+    fld.d      $f25,   $sp,   56
+    fld.d      $f26,   $sp,   64
+    fld.d      $f27,   $sp,   72
+    fld.d      $f28,   $sp,   80
+    fld.d      $f29,   $sp,   88
+    fld.d      $f30,   $sp,   96
+    fld.d      $f31,   $sp,   104
 
     addi.d     $sp,    $sp,   128
     jirl       $r0,    $r1,   0x0

kernel/loongarch64/zgemm_kernel_4x4_lsx.S

@@ -176,15 +176,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     SDARG      $r25,   $sp,   16
     SDARG      $r26,   $sp,   24
     SDARG      $r27,   $sp,   32
-    ST         $f23,   $sp,   40
-    ST         $f24,   $sp,   48
-    ST         $f25,   $sp,   56
-    ST         $f26,   $sp,   64
-    ST         $f27,   $sp,   72
-    ST         $f28,   $sp,   80
-    ST         $f29,   $sp,   88
-    ST         $f30,   $sp,   96
-    ST         $f31,   $sp,   104
+    fst.d      $f23,   $sp,   40
+    fst.d      $f24,   $sp,   48
+    fst.d      $f25,   $sp,   56
+    fst.d      $f26,   $sp,   64
+    fst.d      $f27,   $sp,   72
+    fst.d      $f28,   $sp,   80
+    fst.d      $f29,   $sp,   88
+    fst.d      $f30,   $sp,   96
+    fst.d      $f31,   $sp,   104
     ST         ALPHA_R,$sp,   112
     ST         ALPHA_I,$sp,   120
 
@@ -2300,15 +2300,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     LDARG      $r25,   $sp,   16
     LDARG      $r26,   $sp,   24
     LDARG      $r27,   $sp,   32
-    LD         $f23,   $sp,   40
-    LD         $f24,   $sp,   48
-    LD         $f25,   $sp,   56
-    LD         $f26,   $sp,   64
-    LD         $f27,   $sp,   72
-    LD         $f28,   $sp,   80
-    LD         $f29,   $sp,   88
-    LD         $f30,   $sp,   96
-    LD         $f31,   $sp,   104
+    fld.d      $f23,   $sp,   40
+    fld.d      $f24,   $sp,   48
+    fld.d      $f25,   $sp,   56
+    fld.d      $f26,   $sp,   64
+    fld.d      $f27,   $sp,   72
+    fld.d      $f28,   $sp,   80
+    fld.d      $f29,   $sp,   88
+    fld.d      $f30,   $sp,   96
+    fld.d      $f31,   $sp,   104
 
     addi.d     $sp,    $sp,   128
     jirl       $r0,    $r1,   0x0

kernel/power/KERNEL.POWER5

@@ -59,3 +59,11 @@ CROTKERNEL = ../arm/zrot.c
 ZROTKERNEL = ../arm/zrot.c
 SGEMVNKERNEL = ../arm/gemv_n.c
 SGEMVTKERNEL = ../arm/gemv_t.c
+
+SSCALKERNEL = ../arm/scal.c
+DSCALKERNEL = ../arm/scal.c
+CSCALKERNEL = ../arm/zscal.c
+ZSCALKERNEL = ../arm/zscal.c
+
+
+

kernel/power/casum_microk_power8.c

@@ -159,7 +159,7 @@ static float casum_kernel_16 (long n, float *x)
        "=wa" (t2),	// 5
        "=wa" (t3)	// 6
      :
-       "m" (*x),
+       "m" (*(const float (*)[n * 2]) x),
        "b" (16),	// 8
        "b" (32),	// 9
        "b" (48),	// 10

kernel/power/caxpy_microk_power10.c

@@ -199,14 +199,14 @@ static void caxpy_kernel_8 (long n, float *x, float *y,
 
      "#n=%1 x=%5=%2 y=%0=%3 alpha=(%7,%8) mvecp=%6=%9 ytmp=%4\n"
      :
-       "+m" (*y),
+       "+m" (*(float (*)[n * 2]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y),	// 3
        "=b" (ytmp)	// 4 
      :
-       "m" (*x),
-       "m" (*mvecp),
+       "m" (*(const float (*)[n * 2]) x),
+       "m" (*(const float (*)[2]) mvecp),
        "d" (alpha_r),	// 7
        "d" (alpha_i),	// 8
        "4" (mvecp),	// 9

kernel/power/ccopy_microk_power8.c

@@ -134,12 +134,12 @@ static void ccopy_kernel_32 (long n, float *x, float *y)
 
      "#n=%1 x=%4=%2 y=%0=%3 o16=%5 o32=%6 o48=%7 o64=%8 o80=%9 o96=%10 o112=%11"
      :
-       "=m" (*y),
+       "=m" (*(float (*)[n * 2]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y)		// 3
      :
-       "m" (*x),
+       "m" (*(const float (*)[n * 2]) x),
        "b" (16),	// 5
        "b" (32),	// 6
        "b" (48),	// 7

kernel/power/cdot_microk_power10.c

@@ -166,13 +166,13 @@ static void cdot_kernel_8 (long n, float *x, float *y, float *dot)
 
      "#n=%1 x=%4=%2 y=%5=%3 dot=%0=%6"
      :
-       "=m" (*dot),
+       "=m" (*(float (*)[4]) dot),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y)		// 3
      :
-       "m" (*x),
-       "m" (*y),
+       "m" (*(float (*)[n * 2]) x),
+       "m" (*(float (*)[n * 2]) y),
        "b" (dot),	// 6
        "wa" (mask)
      :

kernel/power/copy_microk_power10.c

@@ -241,12 +241,12 @@ static void copy_kernel (BLASLONG n, FLOAT *x, FLOAT *y)
 
      "#n=%1 x=%4=%2 y=%0=%3"
      :
-       "=m" (*y),
+       "=m" (*(FLOAT (*)[n]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y) 	// 3
      :
-       "m" (*x)
+       "m" (*(FLOAT (*)[n]) x)
      :
        "cr0",
        "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",

kernel/power/cscal_microk_power10.c

@@ -204,7 +204,7 @@ static void zscal_kernel_8 (long n, float *x, float alpha_r, float alpha_i)
 
      "#n=%1 x=%0=%2 alpha=(%3,%4)\n"
      :
-       "+m" (*x),
+       "+m" (*(float (*)[n * 2]) x),
        "+r" (n),	// 1
        "+b" (x)		// 2
      :

kernel/power/cswap_microk_power10.c

@@ -155,8 +155,8 @@ static void cswap_kernel_32 (long n, float *x, float *y)
 
      "#n=%2 x=%0=%3 y=%1=%4"
      :
-       "+m" (*x),
-       "+m" (*y),
+       "+m" (*(FLOAT (*)[n * 2]) x),
+       "+m" (*(FLOAT (*)[n * 2]) y),
        "+r" (n),	// 2
        "+b" (x),	// 3
        "+b" (y)		// 4

kernel/power/cswap_microk_power8.c

@@ -135,8 +135,8 @@ static void cswap_kernel_32 (long n, float *x, float *y)
 
      "#n=%2 x=%0=%3 y=%1=%4 o16=%5 o32=%6 o48=%7 o64=%8 o80=%9 o96=%10 o112=%11"
      :
-       "+m" (*x),
-       "+m" (*y),
+       "+m" (*(float (*)[n * 2]) x),
+       "+m" (*(float (*)[n * 2]) y),
        "+r" (n),	// 2
        "+b" (x),	// 3
        "+b" (y)		// 4

kernel/power/dasum_microk_power10.c

@@ -225,7 +225,7 @@ static double dasum_kernel_16 (long n, double *x)
        "=wa" (a6),	// 17
        "=wa" (a7)	// 18
      :
-       "m" (*x)
+       "m" (*(const double (*)[n]) x)
      :
        "cr0",
        "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",

kernel/power/dasum_microk_power8.c

@@ -154,7 +154,7 @@ static double dasum_kernel_16 (long n, double *x)
        "=wa" (t2),	// 5
        "=wa" (t3)	// 6
      :
-       "m" (*x),
+       "m" (*(const double (*)[n]) x),
        "b" (16),	// 8
        "b" (32),	// 9
        "b" (48),	// 10

kernel/power/daxpy_microk_power10.c

@@ -150,13 +150,13 @@ static void daxpy_kernel_8 (long n, double *x, double *y, double alpha)
 
      "#n=%1 x=%5=%2 y=%0=%3 alpha=%6 t0=%x4\n"
      :
-       "+m" (*y),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y),	// 3
        "=wa" (t0)	// 4
      :
-       "m" (*x),
+       "m" (*(const double (*)[n]) x),
        "d" (alpha)	// 6 
      :
        "cr0",

kernel/power/daxpy_microk_power8.c

@@ -181,7 +181,7 @@ static void daxpy_kernel_8 (long n, double *x, double *y, double alpha)
      "#n=%1 x=%21=%2 y=%0=%3 alpha=%22 o16=%23 o32=%24 o48=%25\n"
      "#t0=%x4 t1=%x5 t2=%x6 t3=%x7 t4=%x8 t5=%x9 t6=%x10 t7=%x11 t8=%x12 t9=%x13 t10=%x14 t11=%x15 t12=%x16 t13=%x17 t14=%x18 t15=%x19 t16=%x20"
      :
-       "+m" (*y),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y),	// 3
@@ -203,7 +203,7 @@ static void daxpy_kernel_8 (long n, double *x, double *y, double alpha)
        "=wa" (t15),	// 19
        "=wa" (t16)	// 20
      :
-       "m" (*x),
+       "m" (*(const double (*)[n]) x),
        "d" (alpha),	// 22
        "b" (16),	// 23
        "b" (32),	// 24

kernel/power/dcopy_microk_power8.c

@@ -134,12 +134,12 @@ static void dcopy_kernel_32 (long n, double *x, double *y)
 
      "#n=%1 x=%4=%2 y=%0=%3 o16=%5 o32=%6 o48=%7 o64=%8 o80=%9 o96=%10 o112=%11"
      :
-       "=m" (*y),
+       "=m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "+b" (y)		// 3
      :
-       "m" (*x),
+       "m" (*(const double (*)[n]) x),
        "b" (16),	// 5
        "b" (32),	// 6
        "b" (48),	// 7

kernel/power/ddot_microk_power10.c

@@ -118,8 +118,8 @@ static double ddot_kernel_8 (long n, double *x, double *y)
        "+b" (x),	// 2
        "+b" (y)		// 3
      :
-       "m" (*x),
-       "m" (*y)
+       "m" (*(const double (*)[n]) x),
+       "m" (*(const double (*)[n]) y)
      :
        "cr0",
        "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",

kernel/power/ddot_microk_power8.c

@@ -151,8 +151,8 @@ static double ddot_kernel_8 (long n, double *x, double *y)
        "=wa" (t2),	// 6
        "=wa" (t3)	// 7
      :
-       "m" (*x),
-       "m" (*y),
+       "m" (*(const double (*)[n]) x),
+       "m" (*(const double (*)[n]) y),
        "b" (16),	// 10
        "b" (32),	// 11
        "b" (48),	// 12

kernel/power/dgemm_kernel_power10.c

@@ -92,7 +92,18 @@ typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));
 	  rowC = (v4sf_t *) &CO[1* ldc+J]; \
           rowC[0] += result[1] * alpha;
 #endif
-
+#define KERNEL(i) \
+          rowA = (vec_t *)&AO[i<< 3];\
+          rowB = *((__vector_pair *)((void *)&BO[i << 3]));\
+          rowB1 = *((__vector_pair *)((void *)&BO[(i << 3) + 4]));\
+          __builtin_mma_xvf64gerpp(&acc0, rowB, rowA[0]);\
+          __builtin_mma_xvf64gerpp(&acc1, rowB1, rowA[0]);\
+          __builtin_mma_xvf64gerpp(&acc2, rowB, rowA[1]);\
+          __builtin_mma_xvf64gerpp(&acc3, rowB1, rowA[1]);\
+          __builtin_mma_xvf64gerpp(&acc4, rowB, rowA[2]);\
+          __builtin_mma_xvf64gerpp(&acc5, rowB1, rowA[2]);\
+          __builtin_mma_xvf64gerpp(&acc6, rowB, rowA[3]);\
+          __builtin_mma_xvf64gerpp(&acc7, rowB1, rowA[3]);
 #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");
 
 #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
@@ -188,7 +199,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
 	  v4sf_t *rowC;
 	  v4sf_t result[4];
 	  __vector_quad acc0, acc1, acc2, acc3, acc4,acc5,acc6,acc7;
-	  BLASLONG l = 0;
+	  BLASLONG l = 1;
 	  vec_t *rowA = (vec_t *) & AO[0];
 	  __vector_pair rowB, rowB1;
 	  rowB = *((__vector_pair *)((void *)&BO[0]));
@@ -201,20 +212,55 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,
 	  __builtin_mma_xvf64ger (&acc5, rowB1, rowA[2]);
 	  __builtin_mma_xvf64ger (&acc6, rowB, rowA[3]);
 	  __builtin_mma_xvf64ger (&acc7, rowB1, rowA[3]);
-	  for (l = 1; l < temp; l++)
-	    {
-	      rowA = (vec_t *) & AO[l << 3];
-	      rowB = *((__vector_pair *)((void *)&BO[l << 3]));
-	      rowB1 = *((__vector_pair *)((void *)&BO[(l << 3) + 4]));
-	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);
-	      __builtin_mma_xvf64gerpp (&acc1, rowB1, rowA[0]);
-	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[1]);
-	      __builtin_mma_xvf64gerpp (&acc3, rowB1, rowA[1]);
-	      __builtin_mma_xvf64gerpp (&acc4, rowB, rowA[2]);
-	      __builtin_mma_xvf64gerpp (&acc5, rowB1, rowA[2]);
-	      __builtin_mma_xvf64gerpp (&acc6, rowB, rowA[3]);
-	      __builtin_mma_xvf64gerpp (&acc7, rowB1, rowA[3]);
-	    }
+	  for (l = 1; l + 15 < temp; l += 16)
+	{
+		 KERNEL (l);
+		 KERNEL (l+1);
+		 KERNEL (l+2);
+		 KERNEL (l+3);
+		 KERNEL (l+4);
+		 KERNEL (l+5);
+		 KERNEL (l+6);
+		 KERNEL (l+7);
+		 KERNEL (l+8);
+		 KERNEL (l+9);
+		 KERNEL (l+10);
+		 KERNEL (l+11);
+		 KERNEL (l+12);
+		 KERNEL (l+13);
+		 KERNEL (l+14);
+		 KERNEL (l+15);
+	}
+	if ((temp - l) & 8)
+	{
+		KERNEL(l);
+		KERNEL(l+1);
+		KERNEL(l+2);
+		KERNEL(l+3);
+		KERNEL(l+4);
+		KERNEL(l+5);
+		KERNEL(l+6);
+		KERNEL(l+7);
+                l += 8;
+	}
+	if ((temp - l) & 4)
+	{
+		KERNEL(l);
+		KERNEL(l+1);
+		KERNEL(l+2);
+		KERNEL(l+3);
+        	l += 4;
+	}
+	if ((temp - l) & 2)
+	{
+		KERNEL(l);
+		KERNEL(l+1);
+		l += 2;
+	}
+	if ((temp - l) & 1)
+	{
+		KERNEL(l);
+	}
 	  SAVE_ACC (&acc0, 0);
 	  SAVE_ACC1 (&acc1, 0);
 	  SAVE_ACC (&acc2, 2);

kernel/power/dgemv_n.c

@@ -147,7 +147,7 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLO
 	BLASLONG m3;
 	BLASLONG n2;
 	BLASLONG lda4 =  lda << 2;
-	volatile FLOAT xbuffer[8] __attribute__ ((aligned (16)));
+	FLOAT xbuffer[8] __attribute__ ((aligned (16)));
 	FLOAT *ybuffer;
 
         if ( m < 1 ) return(0);

kernel/power/dgemv_n_microk_power10.c

@@ -25,8 +25,53 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *****************************************************************************/
 
+#define HAVE_KERNEL_4x2 1
+#define HAVE_KERNEL_4x1 1
 #define HAVE_KERNEL_4x4 1
 
+static void dgemv_kernel_4x2(BLASLONG n, FLOAT *a0, FLOAT *a1, FLOAT *xo, FLOAT *y, FLOAT alpha)
+{
+    FLOAT x0,x1;
+    x0 = xo[0] * alpha;
+    x1 = xo[1] * alpha;
+    __vector double   v_x0 = {x0,x0};
+    __vector double   v_x1 = {x1,x1};
+    __vector double* v_y =(__vector double*)y;
+    __vector double* va0 = (__vector double*)a0;
+    __vector double* va1 = (__vector double*)a1;
+         for (int i=0; i< n/2; i+=2)
+        {
+
+                 v_y[i]+= va0[i]   * v_x0 + va1[i]   * v_x1;
+                 v_y[i+1]+= va0[i+1] * v_x0 + va1[i+1] * v_x1;
+
+        }
+
+
+}
+
+static void dgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *xo, FLOAT *y, FLOAT alpha)
+{
+    BLASLONG i;
+    FLOAT x[1]  __attribute__ ((aligned (16)));
+
+    FLOAT x0,x1;
+    x0 = xo[0] * alpha;
+
+    __vector double   v_x0 = {x0,x0};
+    __vector double* v_y =(__vector double*)y;
+    __vector double* va0 = (__vector double*)a0;
+    for (int i=0; i< n/2; i+=2)
+    {
+
+                 v_y[i]+= va0[i]   * v_x0 ;
+                 v_y[i+1]+= va0[i+1] * v_x0 ;
+
+     }
+
+}
+
+
 static void dgemv_kernel_4x4 (long n, double *ap, long lda, double *x, double *y, double alpha)
 {
   double *a0;
@@ -246,7 +291,7 @@ static void dgemv_kernel_4x4 (long n, double *ap, long lda, double *x, double *y
      "#n=%1 ap=%8=%12 lda=%13 x=%7=%10 y=%0=%2 alpha=%9 o16=%11\n"
      "#a0=%3 a1=%4 a2=%5 a3=%6"
      :
-       "+m" (*y),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (y),	// 2
        "=b" (a0),	// 3
@@ -254,8 +299,8 @@ static void dgemv_kernel_4x4 (long n, double *ap, long lda, double *x, double *y
        "=&b" (a2),	// 5
        "=&b" (a3)	// 6
      :
-       "m" (*x),
-       "m" (*ap),
+       "m" (*(double (*)[4]) x),
+       "m" (*(double (*)[4*n]) ap),
        "d" (alpha),	// 9
        "r" (x),		// 10
        "b" (16),	// 11
@@ -452,7 +497,7 @@ static void dgemv_kernel_4x8 (long n, double *ap, long lda, double *x, double *y
        "stxvp		36, 0( %2)	\n\t"	// y0, y1
 
      :
-       "+m" (*y),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (y),	// 2
        "=b" (a0),	// 3
@@ -465,8 +510,8 @@ static void dgemv_kernel_4x8 (long n, double *ap, long lda, double *x, double *y
        "=&b" (a7),	// 10
        "=b" (tmp)
      :
-       "m" (*x),
-       "m" (*ap),
+       "m" (*(double (*)[4]) x),
+       "m" (*(double (*)[4]) ap),
        "d" (alpha),	// 14
        "r" (x),		// 15
        "3" (ap),	// 16

kernel/power/dgemv_n_microk_power8.c

@@ -276,7 +276,7 @@ static void dgemv_kernel_4x4 (long n, double *ap, long lda, double *x, double *y
      "#n=%1 ap=%8=%12 lda=%13 x=%7=%10 y=%0=%2 alpha=%9 o16=%11\n"
      "#a0=%3 a1=%4 a2=%5 a3=%6"
      :
-       "+m" (*y),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 1
        "+b" (y),	// 2
        "=b" (a0),	// 3
@@ -284,8 +284,8 @@ static void dgemv_kernel_4x4 (long n, double *ap, long lda, double *x, double *y
        "=&b" (a2),	// 5
        "=&b" (a3)	// 6
      :
-       "m" (*x),
-       "m" (*ap),
+       "m" (*(double (*)[4]) x),
+       "m" (*(double (*)[4*n]) ap),
        "d" (alpha),	// 9
        "r" (x),		// 10
        "b" (16),	// 11

kernel/power/drot_microk_power10.c

@@ -130,8 +130,8 @@ static void drot_kernel_16 (long n, double *x, double *y, double c, double s)
 
      "#n=%2 x=%0=%3 y=%1=%4 c=%5 s=%6\n"
      :
-       "+m" (*x),
-       "+m" (*y),
+       "+m" (*(double (*)[n]) x),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 2
        "+b" (x),	// 3
        "+b" (y) 	// 4

kernel/power/drot_microk_power8.c

@@ -185,8 +185,8 @@ static void drot_kernel_16 (long n, double *x, double *y, double c, double s)
      "#n=%2 x=%0=%3 y=%1=%4 c=%13 s=%14 o16=%15 o32=%16 o48=%17\n"
      "#t0=%x5 t1=%x6 t2=%x7 t3=%x8 t4=%x9 t5=%x10 t6=%x11 t7=%x12"
      :
-       "+m" (*x),
-       "+m" (*y),
+       "+m" (*(double (*)[n]) x),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 2
        "+b" (x),	// 3
        "+b" (y),	// 4

kernel/power/dscal_microk_power10.c

@@ -118,7 +118,7 @@ static void dscal_kernel_8 (long n, double *x, double alpha)
 
      "#n=%1 alpha=%3 x=%0=%2"
      :
-       "+m" (*x),
+       "+m" (*(double (*)[n]) x),
        "+r" (n),	// 1
        "+b" (x)		// 2
      :
@@ -158,7 +158,7 @@ static void dscal_kernel_8_zero (long n, double *x)
 
      "#n=%1 x=%0=%2 "
      :
-       "=m" (*x),
+       "=m" (*(double (*)[n]) x),
        "+r" (n),	// 1
        "+b" (x)	// 2
      :

kernel/power/dscal_microk_power8.c

@@ -118,7 +118,7 @@ static void dscal_kernel_8 (long n, double *x, double alpha)
 
      "#n=%1 alpha=%3 x=%0=%2 o16=%4 o32=%5 o48=%6 o64=%7 o80=%8 o96=%9 o112=%10"
      :
-       "+m" (*x),
+       "+m" (*(double (*)[n]) x),
        "+r" (n),	// 1
        "+b" (x)		// 2
      :
@@ -165,7 +165,7 @@ static void dscal_kernel_8_zero (long n, double *x)
 
      "#n=%1 x=%0=%2 t0=%x3 o16=%4 o32=%5 o48=%6 o64=%7 o80=%8 o96=%9 o112=%10"
      :
-       "=m" (*x),
+       "=m" (*(double (*)[n]) x),
        "+r" (n),	// 1
        "+b" (x),	// 2
        "=wa" (t0)	// 3

kernel/power/dswap_microk_power8.c

@@ -135,8 +135,8 @@ static void dswap_kernel_32 (long n, double *x, double *y)
 
      "#n=%2 x=%0=%3 y=%1=%4 o16=%5 o32=%6 o48=%7 o64=%8 o80=%9 o96=%10 o112=%11"
      :
-       "+m" (*x),
-       "+m" (*y),
+       "+m" (*(double (*)[n]) x),
+       "+m" (*(double (*)[n]) y),
        "+r" (n),	// 2
        "+b" (x),	// 3
        "+b" (y)		// 4