Modified assembly label name, so that they are hidden.

Added license informations.

.travis.yml

@@ -24,7 +24,11 @@ before_install:
  - if [[ "$TARGET_BOX" == "WIN64" ]]; then sudo apt-get install -qq binutils-mingw-w64-x86-64 gcc-mingw-w64-x86-64 gfortran-mingw-w64-x86-64; fi
  - if [[ "$TARGET_BOX" == "LINUX32" ]]; then sudo apt-get install -qq  gcc-multilib gfortran-multilib; fi
 
-script: make QUIET_MAKE=1 DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32 $BTYPE
+script: 
+ - make QUIET_MAKE=1 DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32 $BTYPE
+ - if [ "$TARGET_BOX" == "LINUX32" ] || [ "$TARGET_BOX" == "LINUX64" ]; then make -C test DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32 $BTYPE; fi
+ - if [ "$TARGET_BOX" == "LINUX32" ] || [ "$TARGET_BOX" == "LINUX64" ]; then make -C ctest DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32 $BTYPE; fi
+ - if [ "$TARGET_BOX" == "LINUX32" ] || [ "$TARGET_BOX" == "LINUX64" ]; then make -C utest DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32 $BTYPE; fi
 
 # whitelist
 branches:

CONTRIBUTORS.md

@@ -121,6 +121,11 @@ In chronological order:
   * [2014-10-10] trmm and sgemm kernels (optimized for APM's X-Gene 1).
                  ARMv8 support.
 
+* Jerome Robert <jeromerobert@gmx.com>
+  * [2015-01-01] Speed-up small `ger` and `gemv` using stack allocation (bug #478)
+  * [2015-12-23] `stack_check` in `gemv.c` (bug #722)
+  * [2016-01-19] Disable multi-threading in `ger` and `swap` for small matrices (bug #731)
+
 * Dan Kortschak
   * [2015-01-07] Added test for drotmg bug #484.
 

Makefile

@@ -7,10 +7,6 @@ ifneq ($(DYNAMIC_ARCH), 1)
 BLASDIRS += kernel
 endif
 
-ifdef UTEST_CHECK
-SANITY_CHECK = 1
-endif
-
 ifdef SANITY_CHECK
 BLASDIRS += reference
 endif
@@ -85,7 +81,7 @@ endif
 
 shared :
 ifndef NO_SHARED
-ifeq ($(OSNAME), Linux)
+ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS))
 	@$(MAKE) -C exports so
 	@-ln -fs $(LIBSONAME) $(LIBPREFIX).so
 	@-ln -fs $(LIBSONAME) $(LIBPREFIX).so.$(MAJOR_VERSION)

Makefile.arm

@@ -11,8 +11,8 @@ endif
 
 ifeq ($(CORE), ARMV7)
 ifeq ($(OSNAME), Android)
-CCOMMON_OPT += -marm -mfpu=neon  -mfloat-abi=hard -march=armv7-a
-FCOMMON_OPT += -marm -mfpu=neon  -mfloat-abi=hard -march=armv7-a
+CCOMMON_OPT += -marm -mfpu=neon  -mfloat-abi=hard -march=armv7-a -Wl,--no-warn-mismatch
+FCOMMON_OPT += -marm -mfpu=neon  -mfloat-abi=hard -march=armv7-a -Wl,--no-warn-mismatch
 else
 CCOMMON_OPT += -marm -mfpu=vfpv3  -mfloat-abi=hard -march=armv7-a
 FCOMMON_OPT += -marm -mfpu=vfpv3  -mfloat-abi=hard -march=armv7-a
@@ -29,5 +29,3 @@ ifeq ($(CORE), ARMV5)
 CCOMMON_OPT += -marm -march=armv5
 FCOMMON_OPT += -marm -march=armv5
 endif
-
-

Makefile.install

@@ -29,7 +29,7 @@ install : 	lib.grd
 #for inc
 	@echo \#ifndef OPENBLAS_CONFIG_H > $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
 	@echo \#define OPENBLAS_CONFIG_H >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
-	@awk 'NF {print $$1, "OPENBLAS_"$$2, $$3}' config_last.h >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
+	@$(AWK) 'NF {print $$1, "OPENBLAS_"$$2, $$3}' config_last.h >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
 	@echo \#define OPENBLAS_VERSION \" OpenBLAS $(VERSION) \" >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
 	@cat openblas_config_template.h >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
 	@echo \#endif  \/\* OPENBLAS_CONFIG_H \*\/ >> $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/openblas_config.h
@@ -48,10 +48,10 @@ endif
 
 ifndef NO_LAPACKE
 	@echo Copying LAPACKE header files to $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)
-	@-install -pm644 $(NETLIB_LAPACK_DIR)/lapacke/include/lapacke.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke.h
-	@-install -pm644 $(NETLIB_LAPACK_DIR)/lapacke/include/lapacke_config.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_config.h
-	@-install -pm644 $(NETLIB_LAPACK_DIR)/lapacke/include/lapacke_mangling_with_flags.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_mangling.h
-	@-install -pm644 $(NETLIB_LAPACK_DIR)/lapacke/include/lapacke_utils.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_utils.h
+	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke.h
+	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_config.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_config.h
+	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_mangling_with_flags.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_mangling.h
+	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_utils.h $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_utils.h
 endif
 
 #for install static library
@@ -64,7 +64,7 @@ endif
 #for install shared library
 ifndef NO_SHARED
 	@echo Copying the shared library to $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
-ifeq ($(OSNAME), Linux)
+ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS))
 	@install -pm755 $(LIBSONAME) $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
 	@cd $(DESTDIR)$(OPENBLAS_LIBRARY_DIR) ; \
 	ln -fs $(LIBSONAME) $(LIBPREFIX).so ; \

Makefile.rule

@@ -80,7 +80,7 @@ VERSION = 0.2.16.dev
 # NO_LAPACKE = 1
 
 # If you want to use legacy threaded Level 3 implementation.
- USE_SIMPLE_THREADED_LEVEL3 = 1
+# USE_SIMPLE_THREADED_LEVEL3 = 1
 
 # If you want to drive whole 64bit region by BLAS. Not all Fortran
 # compiler supports this. It's safe to keep comment it out if you
@@ -108,6 +108,10 @@ NO_AFFINITY = 1
 # Don't use parallel make.
 # NO_PARALLEL_MAKE = 1
 
+# Force number of make jobs. The default is the number of logical CPU of the host.
+# This is particularly useful when using distcc
+# MAKE_NB_JOBS = 2
+
 # If you would like to know minute performance report of GotoBLAS.
 # FUNCTION_PROFILE = 1
 
@@ -159,10 +163,11 @@ COMMON_PROF = -pg
 # Build Debug version
 # DEBUG = 1
 
-# Improve GEMV and GER for small matrices by stack allocation.
-# For details, https://github.com/xianyi/OpenBLAS/pull/482
+# Set maximum stack allocation.
+# The default value is 2048. 0 disable stack allocation a may reduce GER and GEMV
+# performance. For details, https://github.com/xianyi/OpenBLAS/pull/482
 #
- MAX_STACK_ALLOC=2048
+# MAX_STACK_ALLOC = 0
 
 # Add a prefix or suffix to all exported symbol names in the shared library.
 # Avoid conflicts with other BLAS libraries, especially when using

Makefile.system

@@ -139,6 +139,10 @@ NO_PARALLEL_MAKE=0
 endif
 GETARCH_FLAGS	+= -DNO_PARALLEL_MAKE=$(NO_PARALLEL_MAKE)
 
+ifdef MAKE_NB_JOBS
+GETARCH_FLAGS += -DMAKE_NB_JOBS=$(MAKE_NB_JOBS)
+endif
+
 ifeq ($(HOSTCC), loongcc)
 GETARCH_FLAGS  += -static
 endif
@@ -292,12 +296,14 @@ endif
 ifneq ($(OSNAME), WINNT)
 ifneq ($(OSNAME), CYGWIN_NT)
 ifneq ($(OSNAME), Interix)
+ifneq ($(OSNAME), Android)
 ifdef SMP
 EXTRALIB   += -lpthread
 endif
 endif
 endif
 endif
+endif
 
 # ifeq logical or
 ifeq ($(OSNAME), $(filter $(OSNAME),WINNT CYGWIN_NT Interix))
@@ -324,7 +330,8 @@ ifdef SANITY_CHECK
 CCOMMON_OPT	+= -DSANITY_CHECK -DREFNAME=$(*F)f$(BU)
 endif
 
-ifdef MAX_STACK_ALLOC
+MAX_STACK_ALLOC ?= 2048
+ifneq ($(MAX_STACK_ALLOC), 0)
 CCOMMON_OPT	+= -DMAX_STACK_ALLOC=$(MAX_STACK_ALLOC)
 endif
 
@@ -374,7 +381,7 @@ FCOMMON_OPT	+= -m128bit-long-double
 endif
 ifeq ($(C_COMPILER), CLANG)
 EXPRECISION	= 1
-CCOMMON_OPT	+= -DEXPRECISION 
+CCOMMON_OPT	+= -DEXPRECISION
 FCOMMON_OPT	+= -m128bit-long-double
 endif
 endif
@@ -388,7 +395,7 @@ endif
 
 ifeq ($(USE_OPENMP), 1)
 
-#check 
+#check
 ifeq ($(USE_THREAD), 0)
 $(error OpenBLAS: Cannot set both USE_OPENMP=1 and USE_THREAD=0. The USE_THREAD=0 is only for building single thread version.)
 endif
@@ -952,17 +959,18 @@ ifeq ($(OSNAME), SunOS)
 TAR	= gtar
 PATCH	= gpatch
 GREP	= ggrep
+AWK	= nawk
 else
 TAR	= tar
 PATCH	= patch
 GREP	= grep
+AWK	= awk
 endif
 
 ifndef MD5SUM
 MD5SUM	= md5sum
 endif
 
-AWK	= awk
 
 REVISION = -r$(VERSION)
 MAJOR_VERSION = $(word 1,$(subst ., ,$(VERSION)))
@@ -980,12 +988,8 @@ COMMON_OPT = -O2
 endif
 
 ifndef FCOMMON_OPT
-ifeq ($(OSNAME), WINNT)
-FCOMMON_OPT = -O0 
-else
 FCOMMON_OPT = -O2 -frecursive
 endif
-endif
 
 
 
@@ -1183,4 +1187,3 @@ SUNPATH		= /opt/sunstudio12.1
 else
 SUNPATH		= /opt/SUNWspro
 endif
-

benchmark/scripts/SCIPY/dsyrk.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import time
+import numpy
+from numpy import zeros
+from numpy.random import randn
+from scipy.linalg import blas
+
+
+def run_dsyrk(N, l):
+
+    A = randn(N, N).astype('float64', order='F')
+    C = zeros((N, N), dtype='float64', order='F')
+
+    start = time.time()
+    for i in range(0, l):
+        blas.dsyrk(1.0, A, c=C, overwrite_c=True)
+    end = time.time()
+
+    timediff = (end - start)
+    mflops = (N * N * N) * l / timediff
+    mflops *= 1e-6
+
+    size = "%dx%d" % (N, N)
+    print("%14s :\t%20f MFlops\t%20f sec" % (size, mflops, timediff))
+
+
+if __name__ == "__main__":
+    N = 128
+    NMAX = 2048
+    NINC = 128
+    LOOPS = 1
+
+    z = 0
+    for arg in sys.argv:
+        if z == 1:
+            N = int(arg)
+        elif z == 2:
+            NMAX = int(arg)
+        elif z == 3:
+            NINC = int(arg)
+        elif z == 4:
+            LOOPS = int(arg)
+
+        z = z + 1
+
+    if 'OPENBLAS_LOOPS' in os.environ:
+        p = os.environ['OPENBLAS_LOOPS']
+        if p:
+            LOOPS = int(p)
+
+    print("From: %d To: %d Step=%d Loops=%d" % (N, NMAX, NINC, LOOPS))
+    print("\tSIZE\t\t\tFlops\t\t\t\t\tTime")
+
+    for i in range(N, NMAX + NINC, NINC):
+        run_dsyrk(i, LOOPS)

benchmark/scripts/SCIPY/ssyrk.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import time
+import numpy
+from numpy import zeros
+from numpy.random import randn
+from scipy.linalg import blas
+
+
+def run_ssyrk(N, l):
+
+    A = randn(N, N).astype('float32', order='F')
+    C = zeros((N, N), dtype='float32', order='F')
+
+    start = time.time()
+    for i in range(0, l):
+        blas.ssyrk(1.0, A, c=C, overwrite_c=True)
+    end = time.time()
+
+    timediff = (end - start)
+    mflops = (N * N * N) * l / timediff
+    mflops *= 1e-6
+
+    size = "%dx%d" % (N, N)
+    print("%14s :\t%20f MFlops\t%20f sec" % (size, mflops, timediff))
+
+
+if __name__ == "__main__":
+    N = 128
+    NMAX = 2048
+    NINC = 128
+    LOOPS = 1
+
+    z = 0
+    for arg in sys.argv:
+        if z == 1:
+            N = int(arg)
+        elif z == 2:
+            NMAX = int(arg)
+        elif z == 3:
+            NINC = int(arg)
+        elif z == 4:
+            LOOPS = int(arg)
+
+        z = z + 1
+
+    if 'OPENBLAS_LOOPS' in os.environ:
+        p = os.environ['OPENBLAS_LOOPS']
+        if p:
+            LOOPS = int(p)
+
+    print("From: %d To: %d Step=%d Loops=%d" % (N, NMAX, NINC, LOOPS))
+    print("\tSIZE\t\t\tFlops\t\t\t\t\tTime")
+
+    for i in range(N, NMAX + NINC, NINC):
+        run_ssyrk(i, LOOPS)

cmake/arch.cmake

@@ -14,12 +14,12 @@ if (${ARCH} STREQUAL "x86" OR ${ARCH} STREQUAL "x86_64")
   if (NOT NO_EXPRECISION)
     if (${F_COMPILER} MATCHES "GFORTRAN")
       # N.B. I'm not sure if CMake differentiates between GCC and LSB -hpa
-      if (${CMAKE_C_COMPILER} STREQUAL "GNU" OR ${CMAKE_C_COMPILER} STREQUAL "LSB")
+      if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB")
         set(EXPRECISION	1)
         set(CCOMMON_OPT "${CCOMMON_OPT} -DEXPRECISION -m128bit-long-double")
         set(FCOMMON_OPT	"${FCOMMON_OPT} -m128bit-long-double")
       endif ()
-      if (${CMAKE_C_COMPILER} STREQUAL "Clang")
+      if (${CMAKE_C_COMPILER_ID} STREQUAL "Clang")
         set(EXPRECISION	1)
         set(CCOMMON_OPT "${CCOMMON_OPT} -DEXPRECISION")
         set(FCOMMON_OPT	"${FCOMMON_OPT} -m128bit-long-double")
@@ -28,35 +28,35 @@ if (${ARCH} STREQUAL "x86" OR ${ARCH} STREQUAL "x86_64")
   endif ()
 endif ()
 
-if (${CMAKE_C_COMPILER} STREQUAL "Intel")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "Intel")
   set(CCOMMON_OPT "${CCOMMON_OPT} -wd981")
 endif ()
 
 if (USE_OPENMP)
 
-  if (${CMAKE_C_COMPILER} STREQUAL "GNU" OR ${CMAKE_C_COMPILER} STREQUAL "LSB")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB")
     set(CCOMMON_OPT "${CCOMMON_OPT} -fopenmp")
   endif ()
 
-  if (${CMAKE_C_COMPILER} STREQUAL "Clang")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "Clang")
     message(WARNING "Clang doesn't support OpenMP yet.")
     set(CCOMMON_OPT "${CCOMMON_OPT} -fopenmp")
   endif ()
 
-  if (${CMAKE_C_COMPILER} STREQUAL "Intel")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "Intel")
     set(CCOMMON_OPT "${CCOMMON_OPT} -openmp")
   endif ()
 
-  if (${CMAKE_C_COMPILER} STREQUAL "PGI")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI")
     set(CCOMMON_OPT "${CCOMMON_OPT} -mp")
   endif ()
 
-  if (${CMAKE_C_COMPILER} STREQUAL "OPEN64")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "OPEN64")
     set(CCOMMON_OPT "${CCOMMON_OPT} -mp")
     set(CEXTRALIB "${CEXTRALIB} -lstdc++")
   endif ()
 
-  if (${CMAKE_C_COMPILER} STREQUAL "PATHSCALE")
+  if (${CMAKE_C_COMPILER_ID} STREQUAL "PATHSCALE")
     set(CCOMMON_OPT "${CCOMMON_OPT} -mp")
   endif ()
 endif ()
@@ -87,7 +87,7 @@ if (${ARCH} STREQUAL "ia64")
   set(BINARY_DEFINED 1)
 
   if (${F_COMPILER} MATCHES "GFORTRAN")
-    if (${CMAKE_C_COMPILER} STREQUAL "GNU")
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU")
       # EXPRECISION	= 1
       # CCOMMON_OPT	+= -DEXPRECISION
     endif ()

cmake/lapack.cmake

@@ -48,18 +48,18 @@ set(SLASRC
   sgbbrd.f sgbcon.f sgbequ.f sgbrfs.f sgbsv.f
   sgbsvx.f sgbtf2.f sgbtrf.f sgbtrs.f sgebak.f sgebal.f sgebd2.f
   sgebrd.f sgecon.f sgeequ.f sgees.f  sgeesx.f sgeev.f  sgeevx.f
-  sgegs.f  sgegv.f  sgehd2.f sgehrd.f sgelq2.f sgelqf.f
-  sgels.f  sgelsd.f sgelss.f sgelsx.f sgelsy.f sgeql2.f sgeqlf.f
-  sgeqp3.f sgeqpf.f sgeqr2.f sgeqr2p.f sgeqrf.f sgeqrfp.f sgerfs.f
+  DEPRECATED/sgegs.f  DEPRECATED/sgegv.f  sgehd2.f sgehrd.f sgelq2.f sgelqf.f
+  sgels.f  sgelsd.f sgelss.f DEPRECATED/sgelsx.f sgelsy.f sgeql2.f sgeqlf.f
+  sgeqp3.f DEPRECATED/sgeqpf.f sgeqr2.f sgeqr2p.f sgeqrf.f sgeqrfp.f sgerfs.f
   sgerq2.f sgerqf.f sgesc2.f sgesdd.f  sgesvd.f sgesvx.f
   sgetc2.f sgetri.f
   sggbak.f sggbal.f sgges.f  sggesx.f sggev.f  sggevx.f
   sggglm.f sgghrd.f sgglse.f sggqrf.f
-  sggrqf.f sggsvd.f sggsvp.f sgtcon.f sgtrfs.f sgtsv.f
+  sggrqf.f DEPRECATED/sggsvd.f DEPRECATED/sggsvp.f sgtcon.f sgtrfs.f sgtsv.f
   sgtsvx.f sgttrf.f sgttrs.f sgtts2.f shgeqz.f
   shsein.f shseqr.f slabrd.f slacon.f slacn2.f
   slaein.f slaexc.f slag2.f  slags2.f slagtm.f slagv2.f slahqr.f
-  slahrd.f slahr2.f slaic1.f slaln2.f slals0.f slalsa.f slalsd.f
+  DEPRECATED/slahrd.f slahr2.f slaic1.f slaln2.f slals0.f slalsa.f slalsd.f
   slangb.f slange.f slangt.f slanhs.f slansb.f slansp.f
   slansy.f slantb.f slantp.f slantr.f slanv2.f
   slapll.f slapmt.f
@@ -69,7 +69,7 @@ set(SLASRC
   slarf.f  slarfb.f slarfg.f slarfgp.f slarft.f slarfx.f slargv.f
   slarrv.f slartv.f
   slarz.f  slarzb.f slarzt.f slasy2.f slasyf.f slasyf_rook.f
-  slatbs.f slatdf.f slatps.f slatrd.f slatrs.f slatrz.f slatzm.f
+  slatbs.f slatdf.f slatps.f slatrd.f slatrs.f slatrz.f DEPRECATED/slatzm.f
   sopgtr.f sopmtr.f sorg2l.f sorg2r.f
   sorgbr.f sorghr.f sorgl2.f sorglq.f sorgql.f sorgqr.f sorgr2.f
   sorgrq.f sorgtr.f sorm2l.f sorm2r.f
@@ -97,7 +97,7 @@ set(SLASRC
   stgsja.f stgsna.f stgsy2.f stgsyl.f stpcon.f stprfs.f stptri.f
   stptrs.f
   strcon.f strevc.f strexc.f strrfs.f strsen.f strsna.f strsyl.f
-  strtrs.f stzrqf.f stzrzf.f sstemr.f
+  strtrs.f DEPRECATED/stzrqf.f stzrzf.f sstemr.f
   slansf.f spftrf.f spftri.f spftrs.f ssfrk.f stfsm.f stftri.f stfttp.f
   stfttr.f stpttf.f stpttr.f strttf.f strttp.f
   sgejsv.f  sgesvj.f  sgsvj0.f  sgsvj1.f
@@ -114,14 +114,14 @@ set(CLASRC
   cbdsqr.f cgbbrd.f cgbcon.f cgbequ.f cgbrfs.f cgbsv.f  cgbsvx.f
   cgbtf2.f cgbtrf.f cgbtrs.f cgebak.f cgebal.f cgebd2.f cgebrd.f
   cgecon.f cgeequ.f cgees.f  cgeesx.f cgeev.f  cgeevx.f
-  cgegs.f  cgegv.f  cgehd2.f cgehrd.f cgelq2.f cgelqf.f
-  cgels.f  cgelsd.f cgelss.f cgelsx.f cgelsy.f cgeql2.f cgeqlf.f cgeqp3.f
-  cgeqpf.f cgeqr2.f cgeqr2p.f cgeqrf.f cgeqrfp.f cgerfs.f
+  DEPRECATED/cgegs.f  DEPRECATED/cgegv.f  cgehd2.f cgehrd.f cgelq2.f cgelqf.f
+  cgels.f  cgelsd.f cgelss.f DEPRECATED/cgelsx.f cgelsy.f cgeql2.f cgeqlf.f cgeqp3.f
+  DEPRECATED/cgeqpf.f cgeqr2.f cgeqr2p.f cgeqrf.f cgeqrfp.f cgerfs.f
   cgerq2.f cgerqf.f cgesc2.f cgesdd.f  cgesvd.f
   cgesvx.f cgetc2.f cgetri.f
   cggbak.f cggbal.f cgges.f  cggesx.f cggev.f  cggevx.f cggglm.f
   cgghrd.f cgglse.f cggqrf.f cggrqf.f
-  cggsvd.f cggsvp.f
+  DEPRECATED/cggsvd.f DEPRECATED/cggsvp.f
   cgtcon.f cgtrfs.f cgtsv.f  cgtsvx.f cgttrf.f cgttrs.f cgtts2.f chbev.f
   chbevd.f chbevx.f chbgst.f chbgv.f  chbgvd.f chbgvx.f chbtrd.f
   checon.f cheev.f  cheevd.f cheevr.f cheevx.f chegs2.f chegst.f
@@ -138,7 +138,7 @@ set(CLASRC
   claed0.f claed7.f claed8.f
   claein.f claesy.f claev2.f clags2.f clagtm.f
   clahef.f clahef_rook.f clahqr.f
-  clahrd.f clahr2.f claic1.f clals0.f clalsa.f clalsd.f clangb.f clange.f clangt.f
+  DEPRECATED/clahrd.f clahr2.f claic1.f clals0.f clalsa.f clalsd.f clangb.f clange.f clangt.f
   clanhb.f clanhe.f
   clanhp.f clanhs.f clanht.f clansb.f clansp.f clansy.f clantb.f
   clantp.f clantr.f clapll.f clapmt.f clarcm.f claqgb.f claqge.f
@@ -149,7 +149,7 @@ set(CLASRC
   clarfx.f clargv.f clarnv.f clarrv.f clartg.f clartv.f
   clarz.f  clarzb.f clarzt.f clascl.f claset.f clasr.f  classq.f
   clasyf.f clasyf_rook.f clatbs.f clatdf.f clatps.f clatrd.f clatrs.f clatrz.f
-  clatzm.f cpbcon.f cpbequ.f cpbrfs.f cpbstf.f cpbsv.f
+  DEPRECATED/clatzm.f cpbcon.f cpbequ.f cpbrfs.f cpbstf.f cpbsv.f
   cpbsvx.f cpbtf2.f cpbtrf.f cpbtrs.f cpocon.f cpoequ.f cporfs.f
   cposv.f  cposvx.f cpstrf.f cpstf2.f
   cppcon.f cppequ.f cpprfs.f cppsv.f  cppsvx.f cpptrf.f cpptri.f cpptrs.f
@@ -166,7 +166,7 @@ set(CLASRC
   ctgexc.f ctgsen.f ctgsja.f ctgsna.f ctgsy2.f ctgsyl.f ctpcon.f
   ctprfs.f ctptri.f
   ctptrs.f ctrcon.f ctrevc.f ctrexc.f ctrrfs.f ctrsen.f ctrsna.f
-  ctrsyl.f ctrtrs.f ctzrqf.f ctzrzf.f cung2l.f cung2r.f
+  ctrsyl.f ctrtrs.f DEPRECATED/ctzrqf.f ctzrzf.f cung2l.f cung2r.f
   cungbr.f cunghr.f cungl2.f cunglq.f cungql.f cungqr.f cungr2.f
   cungrq.f cungtr.f cunm2l.f cunm2r.f cunmbr.f cunmhr.f cunml2.f
   cunmlq.f cunmql.f cunmqr.f cunmr2.f cunmr3.f cunmrq.f cunmrz.f
@@ -186,18 +186,18 @@ set(DLASRC
   dgbbrd.f dgbcon.f dgbequ.f dgbrfs.f dgbsv.f
   dgbsvx.f dgbtf2.f dgbtrf.f dgbtrs.f dgebak.f dgebal.f dgebd2.f
   dgebrd.f dgecon.f dgeequ.f dgees.f  dgeesx.f dgeev.f  dgeevx.f
-  dgegs.f  dgegv.f  dgehd2.f dgehrd.f dgelq2.f dgelqf.f
-  dgels.f  dgelsd.f dgelss.f dgelsx.f dgelsy.f dgeql2.f dgeqlf.f
-  dgeqp3.f dgeqpf.f dgeqr2.f dgeqr2p.f dgeqrf.f dgeqrfp.f dgerfs.f
+  DEPRECATED/dgegs.f  DEPRECATED/dgegv.f  dgehd2.f dgehrd.f dgelq2.f dgelqf.f
+  dgels.f  dgelsd.f dgelss.f DEPRECATED/dgelsx.f dgelsy.f dgeql2.f dgeqlf.f
+  dgeqp3.f DEPRECATED/dgeqpf.f dgeqr2.f dgeqr2p.f dgeqrf.f dgeqrfp.f dgerfs.f
   dgerq2.f dgerqf.f dgesc2.f dgesdd.f  dgesvd.f dgesvx.f
   dgetc2.f dgetri.f
   dggbak.f dggbal.f dgges.f  dggesx.f dggev.f  dggevx.f
   dggglm.f dgghrd.f dgglse.f dggqrf.f
-  dggrqf.f dggsvd.f dggsvp.f dgtcon.f dgtrfs.f dgtsv.f
+  dggrqf.f DEPRECATED/dggsvd.f DEPRECATED/dggsvp.f dgtcon.f dgtrfs.f dgtsv.f
   dgtsvx.f dgttrf.f dgttrs.f dgtts2.f dhgeqz.f
   dhsein.f dhseqr.f dlabrd.f dlacon.f dlacn2.f
   dlaein.f dlaexc.f dlag2.f  dlags2.f dlagtm.f dlagv2.f dlahqr.f
-  dlahrd.f dlahr2.f dlaic1.f dlaln2.f dlals0.f dlalsa.f dlalsd.f
+  DEPRECATED/dlahrd.f dlahr2.f dlaic1.f dlaln2.f dlals0.f dlalsa.f dlalsd.f
   dlangb.f dlange.f dlangt.f dlanhs.f dlansb.f dlansp.f
   dlansy.f dlantb.f dlantp.f dlantr.f dlanv2.f
   dlapll.f dlapmt.f
@@ -207,7 +207,7 @@ set(DLASRC
   dlarf.f  dlarfb.f dlarfg.f dlarfgp.f dlarft.f dlarfx.f
   dlargv.f dlarrv.f dlartv.f
   dlarz.f  dlarzb.f dlarzt.f dlasy2.f dlasyf.f dlasyf_rook.f
-  dlatbs.f dlatdf.f dlatps.f dlatrd.f dlatrs.f dlatrz.f dlatzm.f
+  dlatbs.f dlatdf.f dlatps.f dlatrd.f dlatrs.f dlatrz.f DEPRECATED/dlatzm.f
   dopgtr.f dopmtr.f dorg2l.f dorg2r.f
   dorgbr.f dorghr.f dorgl2.f dorglq.f dorgql.f dorgqr.f dorgr2.f
   dorgrq.f dorgtr.f dorm2l.f dorm2r.f
@@ -235,7 +235,7 @@ set(DLASRC
   dtgsja.f dtgsna.f dtgsy2.f dtgsyl.f dtpcon.f dtprfs.f dtptri.f
   dtptrs.f
   dtrcon.f dtrevc.f dtrexc.f dtrrfs.f dtrsen.f dtrsna.f dtrsyl.f
-  dtrtrs.f dtzrqf.f dtzrzf.f dstemr.f
+  dtrtrs.f DEPRECATED/dtzrqf.f dtzrzf.f dstemr.f
   dsgesv.f dsposv.f dlag2s.f slag2d.f dlat2s.f
   dlansf.f dpftrf.f dpftri.f dpftrs.f dsfrk.f dtfsm.f dtftri.f dtfttp.f
   dtfttr.f dtpttf.f dtpttr.f dtrttf.f dtrttp.f
@@ -251,14 +251,14 @@ set(ZLASRC
   zbdsqr.f zgbbrd.f zgbcon.f zgbequ.f zgbrfs.f zgbsv.f  zgbsvx.f
   zgbtf2.f zgbtrf.f zgbtrs.f zgebak.f zgebal.f zgebd2.f zgebrd.f
   zgecon.f zgeequ.f zgees.f  zgeesx.f zgeev.f  zgeevx.f
-  zgegs.f  zgegv.f  zgehd2.f zgehrd.f zgelq2.f zgelqf.f
-  zgels.f  zgelsd.f zgelss.f zgelsx.f zgelsy.f zgeql2.f zgeqlf.f zgeqp3.f
-  zgeqpf.f zgeqr2.f zgeqr2p.f zgeqrf.f zgeqrfp.f zgerfs.f zgerq2.f zgerqf.f
+  DEPRECATED/zgegs.f  DEPRECATED/zgegv.f  zgehd2.f zgehrd.f zgelq2.f zgelqf.f
+  zgels.f  zgelsd.f zgelss.f DEPRECATED/zgelsx.f zgelsy.f zgeql2.f zgeqlf.f zgeqp3.f
+  DEPRECATED/zgeqpf.f zgeqr2.f zgeqr2p.f zgeqrf.f zgeqrfp.f zgerfs.f zgerq2.f zgerqf.f
   zgesc2.f zgesdd.f zgesvd.f zgesvx.f zgetc2.f
   zgetri.f
   zggbak.f zggbal.f zgges.f  zggesx.f zggev.f  zggevx.f zggglm.f
   zgghrd.f zgglse.f zggqrf.f zggrqf.f
-  zggsvd.f zggsvp.f
+  DEPRECATED/zggsvd.f DEPRECATED/zggsvp.f
   zgtcon.f zgtrfs.f zgtsv.f  zgtsvx.f zgttrf.f zgttrs.f zgtts2.f zhbev.f
   zhbevd.f zhbevx.f zhbgst.f zhbgv.f  zhbgvd.f zhbgvx.f zhbtrd.f
   zhecon.f zheev.f  zheevd.f zheevr.f zheevx.f zhegs2.f zhegst.f
@@ -275,7 +275,7 @@ set(ZLASRC
   zlaed0.f zlaed7.f zlaed8.f
   zlaein.f zlaesy.f zlaev2.f zlags2.f zlagtm.f
   zlahef.f zlahef_rook.f zlahqr.f
-  zlahrd.f zlahr2.f zlaic1.f zlals0.f zlalsa.f zlalsd.f zlangb.f zlange.f
+  DEPRECATED/zlahrd.f zlahr2.f zlaic1.f zlals0.f zlalsa.f zlalsd.f zlangb.f zlange.f
   zlangt.f zlanhb.f
   zlanhe.f
   zlanhp.f zlanhs.f zlanht.f zlansb.f zlansp.f zlansy.f zlantb.f
@@ -288,7 +288,7 @@ set(ZLASRC
   zlarfx.f zlargv.f zlarnv.f zlarrv.f zlartg.f zlartv.f
   zlarz.f  zlarzb.f zlarzt.f zlascl.f zlaset.f zlasr.f
   zlassq.f zlasyf.f zlasyf_rook.f
-  zlatbs.f zlatdf.f zlatps.f zlatrd.f zlatrs.f zlatrz.f zlatzm.f
+  zlatbs.f zlatdf.f zlatps.f zlatrd.f zlatrs.f zlatrz.f DEPRECATED/zlatzm.f
   zpbcon.f zpbequ.f zpbrfs.f zpbstf.f zpbsv.f
   zpbsvx.f zpbtf2.f zpbtrf.f zpbtrs.f zpocon.f zpoequ.f zporfs.f
   zposv.f  zposvx.f zpotrs.f zpstrf.f zpstf2.f
@@ -306,7 +306,7 @@ set(ZLASRC
   ztgexc.f ztgsen.f ztgsja.f ztgsna.f ztgsy2.f ztgsyl.f ztpcon.f
   ztprfs.f ztptri.f
   ztptrs.f ztrcon.f ztrevc.f ztrexc.f ztrrfs.f ztrsen.f ztrsna.f
-  ztrsyl.f ztrtrs.f ztzrqf.f ztzrzf.f zung2l.f
+  ztrsyl.f ztrtrs.f DEPRECATED/ztzrqf.f ztzrzf.f zung2l.f
   zung2r.f zungbr.f zunghr.f zungl2.f zunglq.f zungql.f zungqr.f zungr2.f
   zungrq.f zungtr.f zunm2l.f zunm2r.f zunmbr.f zunmhr.f zunml2.f
   zunmlq.f zunmql.f zunmqr.f zunmr2.f zunmr3.f zunmrq.f zunmrz.f

cmake/lapacke.cmake

@@ -2058,10 +2058,10 @@ endif ()
 # add lapack-netlib folder to the sources
 set(LAPACKE_SOURCES "")
 foreach (LAE_FILE ${LAPACKE_REL_SRC})
-  list(APPEND LAPACKE_SOURCES "${NETLIB_LAPACK_DIR}/lapacke/src/${LAE_FILE}")
+  list(APPEND LAPACKE_SOURCES "${NETLIB_LAPACK_DIR}/LAPACKE/src/${LAE_FILE}")
 endforeach ()
 
-set(lapacke_include_dir "${NETLIB_LAPACK_DIR}/lapacke/include")
+set(lapacke_include_dir "${NETLIB_LAPACK_DIR}/LAPACKE/include")
 execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${lapacke_include_dir}/lapacke_mangling_with_flags.h" "${lapacke_include_dir}/lapacke_mangling.h")
 include_directories(${lapacke_include_dir})
 set_source_files_properties(${LAPACKE_SOURCES} PROPERTIES COMPILE_FLAGS "${LAPACK_CFLAGS}")

common.h

@@ -93,7 +93,7 @@ extern "C" {
 #include <sched.h>
 #endif
 
-#if defined(OS_DARWIN) || defined(OS_FREEBSD) || defined(OS_NETBSD)
+#if defined(OS_DARWIN) || defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_ANDROID)
 #include <sched.h>
 #endif
 
@@ -411,7 +411,7 @@ please https://github.com/xianyi/OpenBLAS/issues/246
 #ifndef ASSEMBLER
 #ifdef OS_WINDOWS
 typedef char env_var_t[MAX_PATH];
-#define readenv(p, n) GetEnvironmentVariable((n), (p), sizeof(p))
+#define readenv(p, n) GetEnvironmentVariable((LPCTSTR)(n), (LPTSTR)(p), sizeof(p))
 #else
 typedef char* env_var_t;
 #define readenv(p, n) ((p)=getenv(n))
@@ -727,6 +727,7 @@ typedef struct {
 #endif
 
 #ifndef ASSEMBLER
+#include "common_stackalloc.h"
 #if 0
 #include "symcopy.h"
 #endif

common_param.h

@@ -1194,8 +1194,6 @@ extern gotoblas_t *gotoblas;
 #define XGEMM_DEFAULT_UNROLL_N 2
 #endif
 
-#define GEMM_THREAD gemm_thread_m
-
 #ifndef GEMM_THREAD
 #define GEMM_THREAD gemm_thread_n
 #endif

common_power.h

@@ -236,7 +236,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define HAVE_PREFETCH
 #endif
 
-#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL)
+#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8)
 #define DCBT_ARG	0
 #else
 #define DCBT_ARG	8
@@ -258,6 +258,13 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define L1_PREFETCH	dcbtst
 #endif
 
+#if defined(POWER8)
+#define L1_DUALFETCH
+#define L1_PREFETCHSIZE (16 + 128 * 100)
+#define L1_PREFETCH	dcbtst
+#endif
+
+#
 #ifndef L1_PREFETCH
 #define L1_PREFETCH	dcbt
 #endif
@@ -790,6 +797,8 @@ Lmcount$lazy_ptr:
 #define BUFFER_SIZE     (  2 << 20)
 #elif defined(PPC440FP2)
 #define BUFFER_SIZE     ( 16 << 20)
+#elif defined(POWER8)
+#define BUFFER_SIZE     ( 64 << 20)
 #else
 #define BUFFER_SIZE     ( 16 << 20)
 #endif

common_stackalloc.h

@@ -0,0 +1,73 @@
+/*******************************************************************************
+Copyright (c) 2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************/
+
+#define STACK_ALLOC_PROTECT
+#ifdef STACK_ALLOC_PROTECT
+// Try to detect stack smashing
+#include <assert.h>
+#define STACK_ALLOC_PROTECT_SET volatile int stack_check = 0x7fc01234;
+#define STACK_ALLOC_PROTECT_CHECK assert(stack_check == 0x7fc01234);
+#else
+#define STACK_ALLOC_PROTECT_SET
+#define STACK_ALLOC_PROTECT_CHECK
+#endif
+
+#if defined(MAX_STACK_ALLOC) && MAX_STACK_ALLOC > 0
+
+/*
+ * Allocate a buffer on the stack if the size is smaller than MAX_STACK_ALLOC.
+ * Stack allocation is much faster than blas_memory_alloc or malloc, particularly
+ * when OpenBLAS is used from a multi-threaded application.
+ * SIZE must be carefully chosen to be:
+ * - as small as possible to maximize the number of stack allocation
+ * - large enough to support all architectures and kernel
+ * Chosing a too small SIZE will lead to a stack smashing.
+ */
+#define STACK_ALLOC(SIZE, TYPE, BUFFER)                                    \
+  /* make it volatile because some function (ex: dgemv_n.S) */             \
+  /* do not restore all register */                                        \
+  volatile int stack_alloc_size = SIZE;                                    \
+  if(stack_alloc_size > MAX_STACK_ALLOC / sizeof(TYPE))                    \
+    stack_alloc_size = 0;                                                  \
+  STACK_ALLOC_PROTECT_SET                                                  \
+  TYPE stack_buffer[stack_alloc_size];                                     \
+  BUFFER = stack_alloc_size ? stack_buffer : (TYPE *)blas_memory_alloc(1);
+#else
+  //Original OpenBLAS/GotoBLAS codes.
+  #define STACK_ALLOC(SIZE, TYPE, BUFFER) BUFFER = (TYPE *)blas_memory_alloc(1)
+#endif
+
+
+#if defined(MAX_STACK_ALLOC) && MAX_STACK_ALLOC > 0
+#define STACK_FREE(BUFFER)    \
+  STACK_ALLOC_PROTECT_CHECK   \
+  if(!stack_alloc_size)       \
+    blas_memory_free(BUFFER);
+#else
+#define STACK_FREE(BUFFER) blas_memory_free(BUFFER)
+#endif
+

common_x86.h

@@ -41,6 +41,10 @@
 
 #ifndef ASSEMBLER
 
+#ifdef C_MSVC
+#include <intrin.h>
+#endif
+
 #define MB
 #define WMB
 
@@ -170,12 +174,13 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 
   if (y <= 1) return x;
 
+#if defined(_MSC_VER) && !defined(__clang__)
+  result = x/y;
+  return result;
+#else
+
   y = blas_quick_divide_table[y];
 
-#if defined(_MSC_VER) && !defined(__clang__)
-  (void*)result;
-  return x*y;
-#else
   __asm__ __volatile__  ("mull %0" :"=d" (result) :"a"(x), "0" (y));
 
   return result;

cpuid_arm.c

@@ -115,6 +115,9 @@ int detect(void)
 	  if (strstr(p, "0xc0f")) {
 	    return CPU_CORTEXA15;
 	  }
+	  if (strstr(p, "0xd07")) {
+	    return CPU_ARMV7;  //ARMV8 on 32-bit
+	  }
 
 	}
 
@@ -158,6 +161,27 @@ int detect(void)
 
 
 	}
+
+  	p = (char *) NULL ;
+  	infile = fopen("/proc/cpuinfo", "r");
+
+	while (fgets(buffer, sizeof(buffer), infile))
+	{
+
+		if ((!strncmp("CPU architecture", buffer, 16)))
+		{
+			p = strchr(buffer, ':') + 2;
+			break;
+      		}
+  	}
+  	fclose(infile);
+  	if(p != NULL) {
+	  if (strstr(p, "8")) {
+	    return CPU_ARMV7;  //ARMV8 on 32-bit
+	  }
+
+	}
+
 #endif
 
 	return CPU_UNKNOWN;

cpuid_power.c

@@ -55,6 +55,7 @@
 #define CPUTYPE_POWER6     5
 #define CPUTYPE_CELL       6
 #define CPUTYPE_PPCG4	   7
+#define CPUTYPE_POWER8     8
 
 char *cpuname[] = {
   "UNKNOWN",
@@ -65,6 +66,7 @@ char *cpuname[] = {
   "POWER6",
   "CELL",
   "PPCG4",
+  "POWER8"
 };
 
 char *lowercpuname[] = {
@@ -76,6 +78,7 @@ char *lowercpuname[] = {
   "power6",
   "cell",
   "ppcg4",
+  "power8"
 };
 
 char *corename[] = {
@@ -87,6 +90,7 @@ char *corename[] = {
   "POWER6",
   "CELL",
   "PPCG4",
+  "POWER8"
 };
 
 int detect(void){
@@ -115,7 +119,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER5", 6)) return CPUTYPE_POWER5;
   if (!strncasecmp(p, "POWER6", 6)) return CPUTYPE_POWER6;
   if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
-  if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER6;
+  if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
 

cpuid_x86.c

@@ -1172,6 +1172,9 @@ int get_cpuname(void){
 #endif
           else
 	    return CPUTYPE_NEHALEM;
+	case 13:
+	  // Avoton
+	    return CPUTYPE_NEHALEM;
         }
         break;
       case 5:
@@ -1229,6 +1232,7 @@ int get_cpuname(void){
       case  2:
 	return CPUTYPE_OPTERON;
       case  1:
+      case  3:
       case 10:
 	return CPUTYPE_BARCELONA;
       case  6:
@@ -1674,6 +1678,9 @@ int get_coretype(void){
 #endif
           else
 	    return CORE_NEHALEM;
+	case 13:
+	  // Avoton
+	    return CORE_NEHALEM;
         }
         break;
       case 5:

ctest/c_cblat3.f

@@ -1365,8 +1365,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL CPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL CPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

ctest/c_cblat3_3m.f

@@ -1365,8 +1365,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL CPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL CPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

ctest/c_dblat3.f

@@ -1335,8 +1335,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL DPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL DPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

ctest/c_sblat3.f

@@ -1339,8 +1339,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL SPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL SPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

ctest/c_zblat2.f

@@ -1350,7 +1350,7 @@
 *
 *                       Call the subroutine.
 *
-                        IF( SNAME( 4: 5 ).EQ.'mv' )THEN
+                        IF( SNAME( 10: 11 ).EQ.'mv' )THEN
                            IF( FULL )THEN
                               IF( TRACE )
      $                           WRITE( NTRA, FMT = 9993 )NC, SNAME,
@@ -1376,7 +1376,7 @@
                               CALL CZTPMV( IORDER, UPLO, TRANS, DIAG,
      $                                    N, AA, XX, INCX )
                            END IF
-                        ELSE IF( SNAME( 4: 5 ).EQ.'sv' )THEN
+                        ELSE IF( SNAME( 10: 11 ).EQ.'sv' )THEN
                            IF( FULL )THEN
                               IF( TRACE )
      $                           WRITE( NTRA, FMT = 9993 )NC, SNAME,
@@ -1465,7 +1465,7 @@
                         END IF
 *
                         IF( .NOT.NULL )THEN
-                           IF( SNAME( 4: 5 ).EQ.'mv' )THEN
+                           IF( SNAME( 10: 11 ).EQ.'mv' )THEN
 *
 *                             Check the result.
 *
@@ -1473,7 +1473,7 @@
      $                                    INCX, ZERO, Z, INCX, XT, G,
      $                                    XX, EPS, ERR, FATAL, NOUT,
      $                                    .TRUE. )
-                           ELSE IF( SNAME( 4: 5 ).EQ.'sv' )THEN
+                           ELSE IF( SNAME( 10: 11 ).EQ.'sv' )THEN
 *
 *                             Compute approximation to original vector.
 *
@@ -1611,7 +1611,7 @@
 *     .. Common blocks ..
       COMMON             /INFOC/INFOT, NOUTC, OK
 *     .. Executable Statements ..
-      CONJ = SNAME( 5: 5 ).EQ.'c'
+      CONJ = SNAME( 11: 11 ).EQ.'c'
 *     Define the number of arguments.
       NARGS = 9
 *

ctest/c_zblat3.f

@@ -1366,8 +1366,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL ZPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL ZPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

ctest/c_zblat3_3m.f

@@ -1366,8 +1366,9 @@
 *
   150 CONTINUE
       WRITE( NOUT, FMT = 9996 )SNAME
-      CALL ZPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
-     $      M, N, ALPHA, LDA, LDB)
+      IF( TRACE )
+     $   CALL ZPRCN3( NTRA, NC, SNAME, IORDER, SIDE, UPLO, TRANSA, DIAG,
+     $         M, N, ALPHA, LDA, LDB)
 *
   160 CONTINUE
       RETURN

driver/level3/CMakeLists.txt

@@ -48,8 +48,7 @@ foreach (float_type ${FLOAT_TYPES})
     # TRANS needs to be set/unset when CONJ is set/unset, so can't use it as a combination
     GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK" 3 "herk_N" false ${float_type})
     GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK;TRANS;CONJ" 3 "herk_C" false ${float_type})
-    GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK;THREADED_LEVEL3" 3 "herk_thread_N" false ${float_type})
-    GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK;THREADED_LEVEL3;TRANS;CONJ" 3 "herk_thread_C" false ${float_type})
+
     # Need to set CONJ for trmm and trsm
     GenerateCombinationObjects("trmm_L.c" "UPPER;UNIT" "L;N" "CONJ" 0 "trmm_LR" false ${float_type})
     GenerateCombinationObjects("trmm_L.c" "UPPER;UNIT" "L;N" "TRANSA;CONJ" 0 "trmm_LC" false ${float_type})
@@ -72,6 +71,10 @@ foreach (float_type ${FLOAT_TYPES})
     GenerateNamedObjects("zher2k_k.c" "HER2K;LOWER;TRANS;CONJ" "her2k_LC" false "" "" false ${float_type})
 
     if (SMP AND NOT USE_SIMPLE_THREADED_LEVEL3)
+      #herk
+      GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK;THREADED_LEVEL3" 3 "herk_thread_N" false ${float_type})
+      GenerateCombinationObjects("zherk_k.c" "LOWER" "U" "HERK;THREADED_LEVEL3;TRANS;CONJ" 3 "herk_thread_C" false ${float_type})
+
       #hemm
       GenerateCombinationObjects("zhemm_k.c" "LOWER" "U" "NN;THREADED_LEVEL3" 0 "hemm_thread_L" false ${float_type})
       GenerateCombinationObjects("zhemm_k.c" "LOWER" "U" "NC;RSIDE;THREADED_LEVEL3" 0 "hemm_thread_R" false ${float_type})

driver/level3/level3.c

@@ -335,7 +335,9 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 
         if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
         else
-          if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+        	if (min_jj >= 2*GEMM_UNROLL_N) min_jj = 2*GEMM_UNROLL_N;
+        	else
+          		if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
 
 
 

driver/level3/level3_thread.c

@@ -367,7 +367,9 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 
 	if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
 	else
-		if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
+		if (min_jj >= 2*GEMM_UNROLL_N) min_jj = 2*GEMM_UNROLL_N;
+		else
+			if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
 
 
 	START_RPCC();

driver/others/blas_server.c

@@ -70,7 +70,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /*********************************************************************/
 
 #include "common.h"
-#if defined(OS_LINUX) || defined(OS_NETBSD) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_NETBSD) || defined(OS_DARWIN) || defined(OS_ANDROID) || defined(OS_SUNOS)
 #include <dlfcn.h>
 #include <signal.h>
 #include <sys/resource.h>
@@ -576,10 +576,12 @@ int blas_thread_init(void){
 	struct rlimit rlim;
         const char *msg = strerror(ret);
         fprintf(STDERR, "OpenBLAS blas_thread_init: pthread_create: %s\n", msg);
+#ifdef RLIMIT_NPROC
         if(0 == getrlimit(RLIMIT_NPROC, &rlim)) {
           fprintf(STDERR, "OpenBLAS blas_thread_init: RLIMIT_NPROC "
                   "%ld current, %ld max\n", (long)(rlim.rlim_cur), (long)(rlim.rlim_max));
         }
+#endif
         if(0 != raise(SIGINT)) {
           fprintf(STDERR, "OpenBLAS blas_thread_init: calling exit(3)\n");
           exit(EXIT_FAILURE);

driver/others/dynamic.c

@@ -261,6 +261,11 @@ static gotoblas_t *get_coretype(void){
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
+	//Intel Avoton
+	if (model == 13) { 
+	  openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK); 
+	  return &gotoblas_NEHALEM;
+	}	
 	return NULL;
       case 5:
 	//Intel Broadwell

driver/others/memory.c

@@ -144,7 +144,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #if defined(_MSC_VER) && !defined(__clang__)
 #define CONSTRUCTOR __cdecl
 #define DESTRUCTOR __cdecl
-#elif defined(OS_DARWIN) && defined(C_GCC)
+#elif (defined(OS_DARWIN) || defined(OS_SUNOS)) && defined(C_GCC)
 #define CONSTRUCTOR	__attribute__ ((constructor))
 #define DESTRUCTOR	__attribute__ ((destructor))
 #else
@@ -169,7 +169,7 @@ void goto_set_num_threads(int num_threads) {};
 
 #else
 
-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_SUNOS)
 #ifndef NO_AFFINITY
 int get_num_procs(void);
 #else
@@ -357,7 +357,9 @@ int openblas_get_num_threads(void) {
 #ifndef SMP
   return 1;
 #else
-  return blas_get_cpu_number();
+  // init blas_cpu_number if needed
+  blas_get_cpu_number();
+  return blas_cpu_number;
 #endif
 }
 

exports/Makefile

@@ -100,9 +100,9 @@ libgoto_hpl.def : gensymbol
 ifeq (, $(SYMBOLPREFIX)$(SYMBOLSUFFIX))
 $(LIBDYNNAME) : ../$(LIBNAME) osx.def
 else
-../$(LIBNAME).renamed : ../$(LIBNAME) objconv.def
-	$(OBJCONV) @objconv.def ../$(LIBNAME) ../$(LIBNAME).renamed
-$(LIBDYNNAME) : ../$(LIBNAME).renamed osx.def
+../$(LIBNAME).osx.renamed : ../$(LIBNAME) objconv.def
+	$(OBJCONV) @objconv.def ../$(LIBNAME) ../$(LIBNAME).osx.renamed
+$(LIBDYNNAME) : ../$(LIBNAME).osx.renamed osx.def
 endif
 ifeq ($(NOFORTRAN), $(filter $(NOFORTRAN),1 2))
 #only build without Fortran
@@ -114,7 +114,7 @@ endif
 dllinit.$(SUFFIX) : dllinit.c
 	$(CC) $(CFLAGS) -c -o $(@F) -s $<
 
-ifeq ($(OSNAME), Linux)
+ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS))
 
 so : ../$(LIBSONAME)
 
@@ -224,7 +224,7 @@ linktest.c : gensymbol ../Makefile.system ../getarch.c
 	perl ./gensymbol linktest  $(ARCH) $(BU) $(EXPRECISION) $(NO_CBLAS) $(NO_LAPACK) $(NO_LAPACKE) $(NEED2UNDERSCORES) $(ONLY_CBLAS) "$(SYMBOLPREFIX)" "$(SYMBOLSUFFIX)" > linktest.c
 
 clean ::
-	@rm -f *.def *.dylib __.SYMDEF*
+	@rm -f *.def *.dylib __.SYMDEF* *.renamed
 
 include ../Makefile.tail
 

exports/gensymbol

@@ -548,7 +548,6 @@
 		slatmt,
 		sorm22,
 		spotrf2,
-		xerbla,
 		zgejsv,
 		zgesvdx,
 		zgesvj,

f_check

@@ -1,5 +1,7 @@
 #!/usr/bin/perl
 
+$hostos   = `uname -s | sed -e s/\-.*//`;    chop($hostos);
+
 #
 # 1. Not specified
 #   1.1 Automatically detect, then check compiler
@@ -272,8 +274,9 @@ if ($link ne "") {
 	}
 
 	if ($flags =~ /^\-Y/) {
+	    next if ($hostos eq 'SunOS');
 	    $linker_L .= "-Wl,". $flags . " ";
-	    }
+        }
 
 	if ($flags =~ /^\-rpath\@/) {
 	    $flags =~ s/\@/\,/g;

getarch.c

@@ -86,7 +86,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <sys/types.h>
 #include <sys/sysctl.h>
 #endif
-#ifdef linux
+#if defined(linux) || defined(__sun__)
 #include <sys/sysinfo.h>
 #include <unistd.h>
 #endif
@@ -552,7 +552,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CORENAME  "POWER5"
 #endif
 
-#if defined(FORCE_POWER6) || defined(FORCE_POWER7) || defined(FORCE_POWER8)
+#if defined(FORCE_POWER6) || defined(FORCE_POWER7)
 #define FORCE
 #define ARCHITECTURE    "POWER"
 #define SUBARCHITECTURE "POWER6"
@@ -565,6 +565,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CORENAME  "POWER6"
 #endif
 
+#if defined(FORCE_POWER8) 
+#define FORCE
+#define ARCHITECTURE    "POWER"
+#define SUBARCHITECTURE "POWER8"
+#define SUBDIRNAME      "power"
+#define ARCHCONFIG   "-DPOWER8 " \
+		     "-DL1_DATA_SIZE=65536 -DL1_DATA_LINESIZE=128 " \
+		     "-DL2_SIZE=4194304 -DL2_LINESIZE=128 " \
+		     "-DDTB_DEFAULT_ENTRIES=128 -DDTB_SIZE=4096 -DL2_ASSOCIATIVE=8 "
+#define LIBNAME   "power8"
+#define CORENAME  "POWER8"
+#endif
+
+
 #ifdef FORCE_PPCG4
 #define FORCE
 #define ARCHITECTURE    "POWER"
@@ -906,7 +920,7 @@ static int get_num_cores(void) {
   size_t len;
 #endif
 
-#ifdef linux
+#if defined(linux) || defined(__sun__)
   //returns the number of processors which are currently online
   return sysconf(_SC_NPROCESSORS_ONLN);
 
@@ -998,7 +1012,9 @@ int main(int argc, char *argv[]){
 #endif
 #endif
 
-#if NO_PARALLEL_MAKE==1
+#ifdef MAKE_NB_JOBS
+    printf("MAKE += -j %d\n", MAKE_NB_JOBS);
+#elif NO_PARALLEL_MAKE==1
     printf("MAKE += -j 1\n");
 #else
 #ifndef OS_WINDOWS

interface/gemv.c

@@ -79,11 +79,9 @@ void NAME(char *TRANS, blasint *M, blasint *N,
   FLOAT alpha = *ALPHA;
   FLOAT beta  = *BETA;
   FLOAT *buffer;
+  int buffer_size;
 #ifdef SMP
   int nthreads;
-  int nthreads_max;
-  int nthreads_avail;
-  double MNK;
 #endif
 
   int (*gemv[])(BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT *, BLASLONG,  FLOAT * , BLASLONG, FLOAT *, BLASLONG, FLOAT *) = {
@@ -134,13 +132,10 @@ void CNAME(enum CBLAS_ORDER order,
 
   FLOAT *buffer;
   blasint lenx, leny;
-  int trans;
+  int trans, buffer_size;
   blasint info, t;
 #ifdef SMP
   int nthreads;
-  int nthreads_max;
-  int nthreads_avail;
-  double MNK;
 #endif
 
   int (*gemv[])(BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT *, BLASLONG,  FLOAT * , BLASLONG, FLOAT *, BLASLONG, FLOAT *) = {
@@ -215,43 +210,20 @@ void CNAME(enum CBLAS_ORDER order,
   if (incx < 0) x -= (lenx - 1) * incx;
   if (incy < 0) y -= (leny - 1) * incy;
 
-#ifdef MAX_STACK_ALLOC
-  // make it volatile because some gemv implementation (ex: dgemv_n.S)
-  // do not restore all register
-  volatile int stack_alloc_size = 0;
-  //for gemv_n and gemv_t, try to allocate on stack
-  stack_alloc_size = m + n;
-#ifdef ALIGNED_ACCESS
-  stack_alloc_size += 3;
-#endif
-  if(stack_alloc_size < 128)
-    //dgemv_n.S require a 128 bytes buffer
-    stack_alloc_size = 128;
-
-  if(stack_alloc_size > MAX_STACK_ALLOC / sizeof(FLOAT))
-    stack_alloc_size = 0;
-
-  FLOAT stack_buffer[stack_alloc_size];
-  buffer = stack_alloc_size ? stack_buffer : (FLOAT *)blas_memory_alloc(1);
-  //  printf("stack_alloc_size=%d\n", stack_alloc_size);
-#else
-  //Original OpenBLAS/GotoBLAS codes.
-  buffer = (FLOAT *)blas_memory_alloc(1);
+  buffer_size = m + n + 128 / sizeof(FLOAT);
+#ifdef WINDOWS_ABI
+  buffer_size += 160 / sizeof(FLOAT) ;
 #endif
+  // for alignment
+  buffer_size = (buffer_size + 3) & ~3;
+  STACK_ALLOC(buffer_size, FLOAT, buffer);
 
 #ifdef SMP
 
-  nthreads_max = num_cpu_avail(2);
-  nthreads_avail = nthreads_max;
-
-  MNK = (double) m * (double) n;
-  if ( MNK <= (24.0 * 24.0  * (double) (GEMM_MULTITHREAD_THRESHOLD*GEMM_MULTITHREAD_THRESHOLD) )  )
-        nthreads_max = 1;
-
-  if ( nthreads_max > nthreads_avail )
-        nthreads = nthreads_avail;
+  if ( 1L * m * n < 2304L * GEMM_MULTITHREAD_THRESHOLD )
+    nthreads = 1;
   else
-        nthreads = nthreads_max;
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif
@@ -266,14 +238,7 @@ void CNAME(enum CBLAS_ORDER order,
   }
 #endif
 
-#ifdef MAX_STACK_ALLOC
-  if(!stack_alloc_size){
-    blas_memory_free(buffer);
-  }
-#else
-    blas_memory_free(buffer);
-#endif
-  
+  STACK_FREE(buffer);
   FUNCTION_PROFILE_END(1, m * n + m + n,  2 * m * n);
 
   IDEBUG_END;

interface/ger.c

@@ -171,19 +171,14 @@ void CNAME(enum CBLAS_ORDER order,
   if (incy < 0) y -= (n - 1) * incy;
   if (incx < 0) x -= (m - 1) * incx;
 
-#ifdef MAX_STACK_ALLOC
-  volatile int stack_alloc_size = m;
-  if(stack_alloc_size > MAX_STACK_ALLOC / sizeof(FLOAT))
-      stack_alloc_size = 0;
-  FLOAT stack_buffer[stack_alloc_size];
-  buffer = stack_alloc_size ? stack_buffer : (FLOAT *)blas_memory_alloc(1);
-#else
-  buffer = (FLOAT *)blas_memory_alloc(1);
-#endif
+  STACK_ALLOC(m, FLOAT, buffer);
 
 #ifdef SMPTEST
-  nthreads = num_cpu_avail(2);
-
+  // Threshold chosen so that speed-up is > 1 on a Xeon E5-2630
+  if(1L * m * n > 2048L * GEMM_MULTITHREAD_THRESHOLD)
+    nthreads = num_cpu_avail(2);
+  else
+    nthreads = 1;
 
   if (nthreads == 1) {
 #endif
@@ -198,11 +193,7 @@ void CNAME(enum CBLAS_ORDER order,
   }
 #endif
 
-#ifdef MAX_STACK_ALLOC
-  if(!stack_alloc_size)
-#endif
-    blas_memory_free(buffer);
-
+  STACK_FREE(buffer);
   FUNCTION_PROFILE_END(1, m * n + m + n, 2 * m * n);
 
   IDEBUG_END;

interface/rotg.c

@@ -95,7 +95,7 @@ void CNAME(FLOAT *DA, FLOAT *DB, FLOAT *C, FLOAT *S){
     s = db / r;
     z = ONE;
     if (ada > adb) z = s;
-    if ((ada < adb) && (c != ZERO)) z = ONE / c;
+    if ((ada <= adb) && (c != ZERO)) z = ONE / c;
 
     *C = c;
     *S = s;

interface/swap.c

@@ -77,12 +77,13 @@ void CNAME(blasint n, FLOAT *x, blasint incx, FLOAT *y, blasint incy){
   if (incy < 0) y -= (n - 1) * incy;
 
 #ifdef SMP
-  nthreads = num_cpu_avail(1);
 
   //disable multi-thread when incx==0 or incy==0
   //In that case, the threads would be dependent.
-  if (incx == 0 || incy == 0)
-	  nthreads = 1;
+  if (incx == 0 || incy == 0 || n < 2097152 * GEMM_MULTITHREAD_THRESHOLD / sizeof(FLOAT))
+    nthreads = 1;
+  else
+    nthreads = num_cpu_avail(1);
 
   if (nthreads == 1) {
 #endif

interface/zgemv.c

@@ -79,9 +79,6 @@ void NAME(char *TRANS, blasint *M, blasint *N,
   FLOAT *buffer;
 #ifdef SMP
   int nthreads;
-  int nthreads_max;
-  int nthreads_avail;
-  double MNK;
 #endif
 
   int (*gemv[])(BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT, FLOAT *, BLASLONG,
@@ -148,9 +145,6 @@ void CNAME(enum CBLAS_ORDER order,
   blasint info, t;
 #ifdef SMP
   int nthreads;
-  int nthreads_max;
-  int nthreads_avail;
-  double MNK;
 #endif
 
   int (*gemv[])(BLASLONG, BLASLONG, BLASLONG, FLOAT, FLOAT, FLOAT *, BLASLONG,
@@ -240,18 +234,10 @@ void CNAME(enum CBLAS_ORDER order,
 
 #ifdef SMP
 
-  nthreads_max = num_cpu_avail(2);
-  nthreads_avail = nthreads_max;
-
-  MNK = (double) m * (double) n;
-  if ( MNK <= ( 256.0  * (double) (GEMM_MULTITHREAD_THRESHOLD * GEMM_MULTITHREAD_THRESHOLD)  ))
-        nthreads_max = 1;
-
-  if ( nthreads_max > nthreads_avail )
-        nthreads = nthreads_avail;
+  if ( 1L * m * n < 1024L * GEMM_MULTITHREAD_THRESHOLD )
+    nthreads = 1;
   else
-        nthreads = nthreads_max;
-
+    nthreads = num_cpu_avail(2);
 
   if (nthreads == 1) {
 #endif

interface/zger.c

@@ -213,7 +213,11 @@ void CNAME(enum CBLAS_ORDER order,
   buffer = (FLOAT *)blas_memory_alloc(1);
 
 #ifdef SMPTEST
-  nthreads = num_cpu_avail(2);
+  // Threshold chosen so that speed-up is > 1 on a Xeon E5-2630
+  if(1L * m * n > 36L * sizeof(FLOAT) * sizeof(FLOAT) * GEMM_MULTITHREAD_THRESHOLD)
+    nthreads = num_cpu_avail(2);
+  else
+    nthreads = 1;
 
   if (nthreads == 1) {
 #endif

kernel/Makefile.L3

@@ -36,6 +36,11 @@ ifeq ($(CORE), HASWELL)
 USE_TRMM = 1
 endif
 
+ifeq ($(CORE), POWER8)
+USE_TRMM = 1
+endif
+
+
 
 
 SKERNELOBJS	+= \

kernel/arm/KERNEL.ARMV6

@@ -1,26 +1,4 @@
-SGEMVNKERNEL = ../arm/gemv_n.c
-SGEMVTKERNEL = ../arm/gemv_t.c
-CGEMVNKERNEL = ../arm/zgemv_n.c
-CGEMVTKERNEL = ../arm/zgemv_t.c
 
-DGEMVNKERNEL = ../arm/gemv_n.c
-DGEMVTKERNEL = ../arm/gemv_t.c
-
-CTRMMKERNEL	= ../generic/ztrmmkernel_2x2.c
-CGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
-CGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
-CGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
-
-#ZTRMMKERNEL	= ../generic/ztrmmkernel_2x2.c
-#ZGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
-#ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
-#ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
-
-
-#STRMMKERNEL	= ../generic/trmmkernel_2x2.c
-#SGEMMKERNEL    =  ../generic/gemmkernel_2x2.c
-#SGEMMONCOPY    =  ../generic/gemm_ncopy_2.c
-#SGEMMOTCOPY    =  ../generic/gemm_tcopy_2.c
 
 
 ###############################################################################
@@ -96,19 +74,19 @@ DSWAPKERNEL  = swap_vfp.S
 CSWAPKERNEL  = swap_vfp.S
 ZSWAPKERNEL  = swap_vfp.S
 
-# BAD SGEMVNKERNEL = gemv_n_vfp.S
-# BAD DGEMVNKERNEL = gemv_n_vfp.S
-# CGEMVNKERNEL = cgemv_n_vfp.S
+SGEMVNKERNEL = gemv_n_vfp.S
+DGEMVNKERNEL = gemv_n_vfp.S
+CGEMVNKERNEL = cgemv_n_vfp.S
 ZGEMVNKERNEL = zgemv_n_vfp.S
 
-# BAD SGEMVTKERNEL = gemv_t_vfp.S
-# BAD DGEMVTKERNEL = gemv_t_vfp.S
-# CGEMVTKERNEL = cgemv_t_vfp.S
+SGEMVTKERNEL = gemv_t_vfp.S
+DGEMVTKERNEL = gemv_t_vfp.S
+CGEMVTKERNEL = cgemv_t_vfp.S
 ZGEMVTKERNEL = zgemv_t_vfp.S
 
 STRMMKERNEL	= strmm_kernel_4x2_vfp.S
 DTRMMKERNEL	= dtrmm_kernel_4x2_vfp.S
-#CTRMMKERNEL	= ctrmm_kernel_2x2_vfp.S
+CTRMMKERNEL	= ctrmm_kernel_2x2_vfp.S
 ZTRMMKERNEL	= ztrmm_kernel_2x2_vfp.S
 
 SGEMMKERNEL    = sgemm_kernel_4x2_vfp.S
@@ -131,9 +109,9 @@ DGEMMOTCOPY    = ../generic/gemm_tcopy_2.c
 DGEMMONCOPYOBJ = dgemm_oncopy.o
 DGEMMOTCOPYOBJ = dgemm_otcopy.o
 
-#CGEMMKERNEL    = cgemm_kernel_2x2_vfp.S
-#CGEMMONCOPY    = cgemm_ncopy_2_vfp.S
-#CGEMMOTCOPY    = cgemm_tcopy_2_vfp.S
+CGEMMKERNEL    = cgemm_kernel_2x2_vfp.S
+CGEMMONCOPY    = cgemm_ncopy_2_vfp.S
+CGEMMOTCOPY    = cgemm_tcopy_2_vfp.S
 CGEMMONCOPYOBJ = cgemm_oncopy.o
 CGEMMOTCOPYOBJ = cgemm_otcopy.o
 

kernel/arm/KERNEL.ARMV7

@@ -1,8 +1,3 @@
-SGEMVNKERNEL = ../arm/gemv_n.c
-SGEMVTKERNEL = ../arm/gemv_t.c
-CGEMVNKERNEL = ../arm/zgemv_n.c
-CGEMVTKERNEL = ../arm/zgemv_t.c
-
 
 #################################################################################
 SAMAXKERNEL  = iamax_vfp.S
@@ -77,14 +72,14 @@ DSCALKERNEL  = scal.c
 CSCALKERNEL  = zscal.c
 ZSCALKERNEL  = zscal.c
 
-# BAD SGEMVNKERNEL = gemv_n_vfp.S
-DGEMVNKERNEL = gemv_n_vfp.S
-#CGEMVNKERNEL = cgemv_n_vfp.S
+SGEMVNKERNEL = gemv_n_vfpv3.S
+DGEMVNKERNEL = gemv_n_vfpv3.S
+CGEMVNKERNEL = cgemv_n_vfp.S
 ZGEMVNKERNEL = zgemv_n_vfp.S
 
-# BAD SGEMVTKERNEL = gemv_t_vfp.S
+SGEMVTKERNEL = gemv_t_vfp.S
 DGEMVTKERNEL = gemv_t_vfp.S
-#CGEMVTKERNEL = cgemv_t_vfp.S
+CGEMVTKERNEL = cgemv_t_vfp.S
 ZGEMVTKERNEL = zgemv_t_vfp.S
 
 STRMMKERNEL  =  strmm_kernel_4x4_vfpv3.S
@@ -92,24 +87,15 @@ DTRMMKERNEL  =  dtrmm_kernel_4x4_vfpv3.S
 CTRMMKERNEL  =  ctrmm_kernel_2x2_vfpv3.S
 ZTRMMKERNEL  =  ztrmm_kernel_2x2_vfpv3.S
 
-#SGEMMKERNEL    =  ../generic/gemmkernel_2x2.c
 SGEMMKERNEL    =  sgemm_kernel_4x4_vfpv3.S
-SGEMMINCOPY    =
-SGEMMITCOPY    =
 SGEMMONCOPY    =  sgemm_ncopy_4_vfp.S
 SGEMMOTCOPY    =  sgemm_tcopy_4_vfp.S
-SGEMMINCOPYOBJ =
-SGEMMITCOPYOBJ =
 SGEMMONCOPYOBJ =  sgemm_oncopy.o
 SGEMMOTCOPYOBJ =  sgemm_otcopy.o
 
 DGEMMKERNEL    =  dgemm_kernel_4x4_vfpv3.S
-DGEMMINCOPY    =
-DGEMMITCOPY    =
 DGEMMONCOPY    =  dgemm_ncopy_4_vfp.S
 DGEMMOTCOPY    =  dgemm_tcopy_4_vfp.S
-DGEMMINCOPYOBJ =
-DGEMMITCOPYOBJ =
 DGEMMONCOPYOBJ = dgemm_oncopy.o
 DGEMMOTCOPYOBJ = dgemm_otcopy.o
 

kernel/arm/asum_vfp.S

@@ -367,12 +367,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	.align 5
 
-#if defined(DOUBLE)
-	vsub.f64                d0 , d0 , d0
-	vsub.f64                d1 , d1 , d1
-#else
-	vsub.f32                s0 , s0 , s0
-	vsub.f32                s1 , s1 , s1
+        movs    r12, #0                                          // clear floating point register
+        vmov    s0, r12
+        vmov    s1, r12
+#if     defined(DOUBLE)
+        vcvt.f64.f32    d0, s0
+        vcvt.f64.f32    d1, s1
 #endif
 
 	cmp	N, #0

kernel/arm/cdot_vfp.S

@@ -185,14 +185,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r4, fp, #128
 	vstm	r4, { s8 - s15} 				// store floating point registers
 
+        movs    r4, #0                                          // clear floating point register
+        vmov    s0, r4
+	vmov	s1, s0
+	vmov	s2, s0
+	vmov	s3, s0
+
 	mov	Y, OLD_Y
 	ldr	INC_Y, OLD_INC_Y
 
-	vsub.f32                s0 , s0 , s0
-	vsub.f32                s1 , s1 , s1
-	vsub.f32                s2 , s2 , s2
-	vsub.f32                s3 , s3 , s3
-
 	cmp	N, #0
 	ble	cdot_kernel_L999
 

kernel/arm/cgemm_kernel_2x2_vfp.S

@@ -57,6 +57,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -138,7 +142,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -340,7 +344,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s12, s8
 	vmov.f32		s13, s8
@@ -514,7 +518,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -681,7 +685,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 
 .endm
@@ -822,6 +826,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 4 * 2
 	str	r3, LDC

kernel/arm/cgemm_kernel_2x2_vfpv3.S

@@ -73,6 +73,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -147,7 +151,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -368,7 +372,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -550,7 +554,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -730,7 +734,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s24, s16
 	vmov.f32		s25, s16
@@ -879,6 +883,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s31} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 4 * 2
 	str	r3, LDC

kernel/arm/cgemv_n_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 #define ALPHA_I [fp, #-236]
 #define ALPHA_R [fp, #-244]
 
@@ -117,7 +121,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .macro INIT_F4
 
 	pld	[ YO, #Y_PRE ]
-        vsub.f32                s8 , s8 , s8
+	flds			s8 , FP_ZERO
         vmov.f32                s9 , s8
         vmov.f32                s10, s8
         vmov.f32                s11, s8
@@ -220,7 +224,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-        vsub.f32                s8 , s8 , s8
+	flds			s8 , FP_ZERO
         vmov.f32                s9 , s8
 
 .endm
@@ -267,7 +271,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S4
 
-        vsub.f32                s8 , s8 , s8
+	flds			s8 , FP_ZERO
         vmov.f32                s9 , s8
         vmov.f32                s10, s8
         vmov.f32                s11, s8
@@ -384,7 +388,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-        vsub.f32                s8 , s8 , s8
+	flds			s8 , FP_ZERO
         vmov.f32                s9 , s8
 
 .endm
@@ -448,6 +452,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	OLD_M, #0
 	ble	cgemvn_kernel_L999
 

kernel/arm/cgemv_t_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 #define N	[fp, #-252 ]
 #define A	[fp, #-256 ]
 
@@ -116,10 +120,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F2
 
-	vsub.f32	s12, s12, s12
-	vsub.f32	s13, s13, s13
-	vsub.f32	s14, s14, s14
-	vsub.f32	s15, s15, s15
+	flds		s12, FP_ZERO
+	vmov.f32	s13, s12
+	vmov.f32	s14, s12
+	vmov.f32	s15, s12
 
 .endm
 
@@ -172,8 +176,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f32	s12, s12, s12
-	vsub.f32	s13, s13, s13
+	flds		s12, FP_ZERO
+	vmov.f32	s13, s12
 
 .endm
 
@@ -215,10 +219,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S2
 
-	vsub.f32	s12, s12, s12
-	vsub.f32	s13, s13, s13
-	vsub.f32	s14, s14, s14
-	vsub.f32	s15, s15, s15
+	flds		s12, FP_ZERO
+	vmov.f32	s13, s12
+	vmov.f32	s14, s12
+	vmov.f32	s15, s12
 
 .endm
 
@@ -281,8 +285,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f32	s12, s12, s12
-	vsub.f32	s13, s13, s13
+	flds		s12, FP_ZERO
+	vmov.f32	s13, s12
 
 .endm
 
@@ -345,6 +349,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	M, #0
 	ble	cgemvt_kernel_L999
 

kernel/arm/ctrmm_kernel_2x2_vfp.S

@@ -59,6 +59,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-232]
+#define FP_ZERO_0 [fp, #-232]
+#define FP_ZERO_1 [fp, #-228]
+
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -136,7 +141,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -301,10 +306,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	flds		s0, ALPHA_R
 	flds		s1, ALPHA_I
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
-	vsub.f32	s6, s6, s6
-	vsub.f32	s7, s7, s7
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
+	vmov.f32	s6, s4
+	vmov.f32	s7, s4
 
 	FMAC_R1 s4 , s0 , s8
 	FMAC_I1 s5 , s0 , s9
@@ -318,10 +323,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	fstmias CO1, { s4 - s7 }
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
-	vsub.f32	s6, s6, s6
-	vsub.f32	s7, s7, s7
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
+	vmov.f32	s6, s4
+	vmov.f32	s7, s4
 
 	FMAC_R1 s4 , s0 , s12
 	FMAC_I1 s5 , s0 , s13
@@ -343,7 +348,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s12, s8
 	vmov.f32		s13, s8
@@ -490,8 +495,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	flds		s0, ALPHA_R
 	flds		s1, ALPHA_I
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
 
 	FMAC_R1 s4 , s0 , s8
 	FMAC_I1 s5 , s0 , s9
@@ -500,8 +505,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	fstmias CO1, { s4 - s5 }
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
 
 	FMAC_R1 s4 , s0 , s12
 	FMAC_I1 s5 , s0 , s13
@@ -519,7 +524,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -663,10 +668,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	flds		s0, ALPHA_R
 	flds		s1, ALPHA_I
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
-	vsub.f32	s6, s6, s6
-	vsub.f32	s7, s7, s7
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
+	vmov.f32	s6, s4
+	vmov.f32	s7, s4
 
 	FMAC_R1 s4 , s0 , s8
 	FMAC_I1 s5 , s0 , s9
@@ -689,7 +694,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 
 .endm
@@ -795,8 +800,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	flds		s0, ALPHA_R
 	flds		s1, ALPHA_I
 
-	vsub.f32	s4, s4, s4
-	vsub.f32	s5, s5, s5
+	flds		s4, FP_ZERO
+	vmov.f32	s5, s4
 
 	FMAC_R1 s4 , s0 , s8
 	FMAC_I1 s5 , s0 , s9
@@ -831,6 +836,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 4 * 2
 	str	r3, LDC

kernel/arm/ctrmm_kernel_2x2_vfpv3.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-236]
+#define FP_ZERO_0 [fp, #-236]
+#define FP_ZERO_1 [fp, #-232]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -134,7 +138,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16 , FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -351,7 +355,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16 , FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -529,7 +533,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16 , FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -706,7 +710,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16 , FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s24, s16
 	vmov.f32		s25, s16
@@ -852,6 +856,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s31} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 4 * 2
 	str	r3, LDC

kernel/arm/ddot_vfp.S

@@ -31,6 +31,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * 	 CTEST			: OK
 * 	 TEST			: OK
 *
+* 2016/01/23 Saar
+*        Bugfix for Refs #750 and #740
 **************************************************************************************/
 
 #define ASSEMBLER
@@ -152,8 +154,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	mov	Y, OLD_Y
 	ldr	INC_Y, OLD_INC_Y
 
-	vsub.f64                d0 , d0 , d0
-	vsub.f64                d1 , d1 , d1
+        movs    r4, #0                                          // clear floating point register
+        vmov    s0, r4
+        vmov    s1, r4
+        vcvt.f64.f32    d0, s0
+        vcvt.f64.f32    d1, s1
+
 
 	cmp	N, #0
 	ble	ddot_kernel_L999

kernel/arm/dgemm_kernel_4x2_vfp.S

@@ -56,8 +56,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA	[fp, #-280]
 
+
 #define B	[fp, #4 ]
 #define C	[fp, #8 ]
 #define OLD_LDC	[fp, #12 ]
@@ -85,7 +90,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -173,7 +178,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d12, d8
 	vmov.f64		d13, d8
@@ -233,7 +238,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 	vmov.f64		d12, d8
 
 .endm
@@ -283,7 +288,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -338,7 +343,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 	vmov.f64		d9 , d8
 
 .endm
@@ -380,7 +385,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8, FP_ZERO
 
 .endm
 
@@ -433,6 +438,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 8
 	str	r3, LDC

kernel/arm/dgemm_kernel_4x4_vfpv3.S

@@ -73,6 +73,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA	[fp, #-280]
 
 #define B	[fp, #4 ]
@@ -102,7 +106,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -376,7 +380,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -470,7 +474,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d20, d16
 	vmov.f64		d24, d16
 	vmov.f64		d28, d16
@@ -533,7 +537,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -617,7 +621,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -678,7 +682,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d20, d16
 
 .endm
@@ -723,7 +727,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -782,7 +786,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 
 .endm
@@ -826,7 +830,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 
 .endm
 
@@ -880,6 +884,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	str	OLD_A, A
 	vstr	OLD_ALPHA, ALPHA
 
+	movs	r4, #0
+	str	r4, FP_ZERO
+	str	r4, FP_ZERO_1
+
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 

kernel/arm/dtrmm_kernel_4x2_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-232]
+#define FP_ZERO_0 [fp, #-232]
+#define FP_ZERO_1 [fp, #-228]
+
 #define ALPHA	[fp, #-276 ]
 
 #define B	[fp, #4 ]
@@ -90,7 +94,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -165,7 +169,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d12, d8
 	vmov.f64		d13, d8
@@ -220,7 +224,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d12, d8
 
 .endm
@@ -268,7 +272,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9, d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -318,7 +322,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 
 .endm
@@ -357,7 +361,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 
 .endm
 
@@ -409,6 +413,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 8
 	str	r3, LDC

kernel/arm/dtrmm_kernel_4x4_vfpv3.S

@@ -59,6 +59,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-236]
+#define FP_ZERO_0 [fp, #-236]
+#define FP_ZERO_1 [fp, #-232]
+
+
 #define ALPHA	[fp, #-276 ]
 
 #define B	[fp, #4 ]
@@ -89,7 +94,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -386,7 +391,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -468,7 +473,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x4
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d20, d16
 	vmov.f64		d24, d16
 	vmov.f64		d28, d16
@@ -527,7 +532,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -601,7 +606,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -656,7 +661,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d20, d16
 
 .endm
@@ -699,7 +704,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -753,7 +758,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 
 .endm
@@ -794,7 +799,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 
 .endm
 
@@ -850,6 +855,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #3					// ldc = ldc * 8
 	str	r3, LDC

kernel/arm/gemv_n_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 #define M	[fp, #-252 ]
 #define A	[fp, #-256 ]
 
@@ -79,7 +83,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	pld     [ YO , #Y_PRE ]
 	pld     [ YO , #Y_PRE+32 ]
 
-	vsub.f64	d8 , d8 , d8
+	fldd		d8 , FP_ZERO
 	vmov.f64	d9  , d8
 	vmov.f64	d10 , d8
 	vmov.f64	d11 , d8
@@ -158,7 +162,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f64	d12 , d12 , d12
+	fldd		d12 , FP_ZERO
 
 .endm
 
@@ -185,7 +189,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S4
 
-	vsub.f64	d12 , d12 , d12
+	fldd		d12 , FP_ZERO
 	vmov.f64	d13 , d12
 	vmov.f64	d14 , d12
 	vmov.f64	d15 , d12
@@ -245,7 +249,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f64	d12 , d12 , d12
+	fldd		d12 , FP_ZERO
 
 .endm
 
@@ -279,7 +283,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	pld     [ YO , #Y_PRE ]
 
-	vsub.f32	s8 , s8 , s8
+	flds		s8 , FP_ZERO
 	vmov.f32	s9  , s8
 	vmov.f32	s10 , s8
 	vmov.f32	s11 , s8
@@ -357,7 +361,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f32	s12 , s12 , s12
+	flds		s12 , FP_ZERO
 
 .endm
 
@@ -384,7 +388,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S4
 
-	vsub.f32	s12 , s12 , s12
+	flds		s12 , FP_ZERO
 	vmov.f32	s13 , s12
 	vmov.f32	s14 , s12
 	vmov.f32	s15 , s12
@@ -445,7 +449,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f32	s12 , s12 , s12
+	flds		s12 , FP_ZERO
 
 .endm
 
@@ -494,6 +498,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	OLD_M, #0
 	ble	gemvn_kernel_L999
 

kernel/arm/gemv_n_vfpv3.S

@@ -62,6 +62,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define M	[fp, #-252 ]
 #define A	[fp, #-256 ]
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 
 #define X_PRE	64
 #define Y_PRE	0
@@ -79,7 +83,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	pld     [ YO , #Y_PRE ]
 	pld     [ YO , #Y_PRE+32 ]
 
-	vsub.f64	d24 , d24 , d24
+	fldd		d24 , FP_ZERO
 	vmov.f64	d25 , d24
 	vmov.f64	d26 , d24
 	vmov.f64	d27 , d24
@@ -147,7 +151,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f64	d24 , d24 , d24
+	fldd		d24 , FP_ZERO
 
 .endm
 
@@ -175,7 +179,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S8
 
-	vsub.f64	d24 , d24 , d24
+	fldd		d24 , FP_ZERO
 	vmov.f64	d25 , d24
 	vmov.f64	d26 , d24
 	vmov.f64	d27 , d24
@@ -269,7 +273,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f64	d24 , d24 , d24
+	fldd		d24 , FP_ZERO
 
 .endm
 
@@ -302,7 +306,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	pld     [ YO , #Y_PRE ]
 
-	vsub.f32	s24 , s24 , s24
+	flds		s24 , FP_ZERO
 	vmov.f32	s25 , s24
 	vmov.f32	s26 , s24
 	vmov.f32	s27 , s24
@@ -368,7 +372,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f32	s24 , s24 , s24
+	flds		s24 , FP_ZERO
 
 .endm
 
@@ -396,7 +400,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S8
 
-	vsub.f32	s24 , s24 , s24
+	flds		s24 , FP_ZERO
 	vmov.f32	s25 , s24
 	vmov.f32	s26 , s24
 	vmov.f32	s27 , s24
@@ -489,7 +493,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f32	s24 , s24 , s24
+	flds		s24 , FP_ZERO
 
 .endm
 
@@ -538,6 +542,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s31 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	OLD_M, #0
 	ble	gemvn_kernel_L999
 

kernel/arm/gemv_t_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 #define N	[fp, #-252 ]
 #define A	[fp, #-256 ]
 
@@ -75,8 +79,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F2
 
-	vsub.f64	d2 , d2 , d2
-	vsub.f64	d3 , d3 , d3
+	fldd		d2, FP_ZERO
+	vmov.f64	d3 , d2 
 
 .endm
 
@@ -123,7 +127,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f64	d2 , d2 , d2
+	fldd		d2, FP_ZERO
+	vmov.f64	d3 , d2 
 
 .endm
 
@@ -160,8 +165,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S2
 
-	vsub.f64	d2 , d2 , d2
-	vsub.f64	d3 , d3 , d3
+	fldd		d2, FP_ZERO
+	vmov.f64	d3 , d2 
 
 .endm
 
@@ -224,7 +229,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f64	d2 , d2 , d2
+	fldd		d2, FP_ZERO
+	vmov.f64	d3 , d2 
 
 .endm
 
@@ -276,8 +282,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F2
 
-	vsub.f32	s2 , s2 , s2
-	vsub.f32	s3 , s3 , s3
+	flds		s2 , FP_ZERO
+	vmov.f32	s3 , s2 
+
 
 .endm
 
@@ -321,7 +328,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f32	s2 , s2 , s2
+	flds		s2 , FP_ZERO
 
 .endm
 
@@ -356,8 +363,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S2
 
-	vsub.f32	s2 , s2 , s2
-	vsub.f32	s3 , s3 , s3
+	flds		s2 , FP_ZERO
+	vmov.f32	s3 , s2 
 
 .endm
 
@@ -418,7 +425,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f32	s2 , s2 , s2
+	flds		s2 , FP_ZERO
 
 .endm
 
@@ -488,6 +495,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	M, #0
 	ble	gemvt_kernel_L999
 

kernel/arm/iamax_vfp.S

@@ -341,11 +341,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	.align 5
 	push    {r4}
 
-#if defined(DOUBLE)
-	vsub.f64                d0 , d0 , d0
-#else
-	vsub.f32                s0 , s0 , s0
+        movs    r12, #0                                          // clear floating point register
+        vmov    s0, r12
+#if     defined(DOUBLE)
+        vcvt.f64.f32    d0, s0
 #endif
+
 	mov	INDEX, #0
 
 	cmp	N, #0

kernel/arm/nrm2_vfp.S

@@ -409,12 +409,20 @@ KERNEL_S1_END_\@:
 
 #if	defined(DOUBLE)
 
+znrm2_zero:
+	.word	0x00000000
+	.word	0x00000000
+
+
 znrm2_one:
 	.word	0x00000000
 	.word	0x3ff00000
 
 #else
 
+cnrm2_zero:
+	.word	0x00000000
+
 cnrm2_one:
 	.word	0x3f800000
 
@@ -424,12 +432,20 @@ cnrm2_one:
 
 #if	defined(DOUBLE)
 
+dnrm2_zero:
+	.word	0x00000000
+	.word	0x00000000
+
+
 dnrm2_one:
 	.word	0x00000000
 	.word	0x3ff00000
 
 #else
 
+snrm2_zero:
+	.word	0x00000000
+
 snrm2_one:
 	.word	0x3f800000
 
@@ -446,12 +462,12 @@ nrm2_begin:
 #if defined(COMPLEX)
 
 #if defined(DOUBLE)
-	vsub.f64                d0 , d0 , d0		// scale=0.0
+	vldr.64			d0 , znrm2_zero	
 	vldr.64			d1 , znrm2_one		// ssq=1.0
 	vmov.f64		d7 , d1			// value 1.0
 	vmov.f64		d6 , d0			// value 0.0
 #else
-	vsub.f32                s0 , s0 , s0		// scale=0.0
+	vldr.32			s0 , cnrm2_zero
 	vldr.32			s1 , cnrm2_one		// ssq=1.0
 	vmov.f32		s7 , s1			// value 1.0
 	vmov.f32		s6 , s0			// value 0.0
@@ -460,12 +476,12 @@ nrm2_begin:
 #else
 
 #if defined(DOUBLE)
-	vsub.f64                d0 , d0 , d0		// scale=0.0
+	vldr.64			d0 , dnrm2_zero	
 	vldr.64			d1 , dnrm2_one		// ssq=1.0
 	vmov.f64		d7 , d1			// value 1.0
 	vmov.f64		d6 , d0			// value 0.0
 #else
-	vsub.f32                s0 , s0 , s0		// scale=0.0
+	vldr.32			s0 , snrm2_zero
 	vldr.32			s1 , snrm2_one		// ssq=1.0
 	vmov.f32		s7 , s1			// value 1.0
 	vmov.f32		s6 , s0			// value 0.0

kernel/arm/nrm2_vfpv3.S

@@ -405,12 +405,15 @@ KERNEL_S1_END_\@:
 	.align 5
 
 #if defined(DOUBLE)
-	vsub.f64                d0 , d0 , d0		// scale=0.0
+	movs			r12 , #0
+	vmov.f32		s0 , r12		// scale=0.0
+	vcvt.f64.f32    d0, s0
 	vmov.f64		d1 , #1.0		// ssq=1.0
 	vmov.f64		d7 , d1			// value 1.0
 	vmov.f64		d6 , d0			// value 0.0
 #else
-	vsub.f32                s0 , s0 , s0		// scale=0.0
+	movs			r12 , #0
+	vmov.f32		s0 , r12		// scale=0.0
 	vmov.f32		s1 , #1.0		// ssq=1.0
 	vmov.f32		s7 , s1			// value 1.0
 	vmov.f32		s6 , s0			// value 0.0

kernel/arm/sdot_vfp.S

@@ -31,6 +31,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * 	 CTEST			: OK (no test for dsdot)
 * 	 TEST			: OK (no test for dsdot)
 *
+* 2016/01/23 Saar
+*	 Bugfix	for Refs #750 and #740
 **************************************************************************************/
 
 #define ASSEMBLER
@@ -240,16 +242,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	mov	Y, OLD_Y
 	ldr	INC_Y, OLD_INC_Y
 
+	movs	r4, #0						// clear floating point register
+	vmov	s0, r4
+	vmov	s1, r4
 #if	defined(DSDOT)
-
-	vsub.f64                d0 , d0 , d0
-	vsub.f64                d1 , d1 , d1
-
-#else
-
-	vsub.f32                s0 , s0 , s0
-	vsub.f32                s1 , s1 , s1
-
+	vcvt.f64.f32    d0, s0
+	vcvt.f64.f32    d1, s1
 #endif
 
 	cmp	N, #0

kernel/arm/sgemm_kernel_4x2_vfp.S

@@ -56,6 +56,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA	[fp, #-280]
 
 #define B	[fp, #4 ]
@@ -85,7 +89,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -161,7 +165,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s12, s8
 	vmov.f32		s13, s8
@@ -221,7 +225,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 	vmov.f32		s12, s8
 
 .endm
@@ -271,7 +275,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -326,7 +330,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 	vmov.f32		s9 , s8
 
 .endm
@@ -368,7 +372,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8, FP_ZERO
 
 .endm
 
@@ -421,6 +425,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #2					// ldc = ldc * 4
 	str	r3, LDC

kernel/arm/sgemm_kernel_4x4_vfpv3.S

@@ -73,7 +73,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
-#define ALPHA	[fp, #-280]
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, #-240]
+#define FP_ZERO_1 [fp, #-236]
+
+#define ALPHA   [fp, #-280]
 
 #define B	[fp, #4 ]
 #define C	[fp, #8 ]
@@ -102,7 +106,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -349,7 +353,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -443,7 +447,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s20, s16
 	vmov.f32		s24, s16
 	vmov.f32		s28, s16
@@ -506,7 +510,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -590,7 +594,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -651,7 +655,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s20, s16
 
 .endm
@@ -696,7 +700,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -755,7 +759,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 	vmov.f32		s17, s16
 
 .endm
@@ -799,7 +803,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			s16, FP_ZERO
 
 .endm
 
@@ -856,6 +860,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s31} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #2					// ldc = ldc * 4
 	str	r3, LDC

kernel/arm/strmm_kernel_4x2_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-232]
+#define FP_ZERO_0 [fp, #-232]
+#define FP_ZERO_1 [fp, #-228]
+
 #define ALPHA	[fp, #-276 ]
 
 #define B	[fp, #4 ]
@@ -90,7 +94,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -156,7 +160,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s12, s8
 	vmov.f32		s13, s8
@@ -211,7 +215,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s12, s8
 
 .endm
@@ -259,7 +263,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9, s8
 	vmov.f32		s10, s8
 	vmov.f32		s11, s8
@@ -309,7 +313,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 	vmov.f32		s9 , s8
 
 .endm
@@ -348,7 +352,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s8 , s8 , s8
+	flds			s8 , FP_ZERO
 
 .endm
 
@@ -400,6 +404,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #2					// ldc = ldc * 4
 	str	r3, LDC

kernel/arm/strmm_kernel_4x4_vfpv3.S

@@ -58,6 +58,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define K	[fp, #-264 ]
 #define A	[fp, #-268 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA	[fp, #-280]
 
 #define B	[fp, #4 ]
@@ -88,7 +92,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -322,7 +326,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -405,7 +409,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x4
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s20, s16
 	vmov.f32		s24, s16
 	vmov.f32		s28, s16
@@ -464,7 +468,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -538,7 +542,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s20, s16
 	vmov.f32		s21, s16
@@ -593,7 +597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s20, s16
 
 .endm
@@ -636,7 +640,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT4x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 	vmov.f32		s18, s16
 	vmov.f32		s19, s16
@@ -690,7 +694,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 	vmov.f32		s17, s16
 
 .endm
@@ -731,7 +735,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f32		s16 , s16 , s16
+	flds			S16, FP_ZERO
 
 .endm
 
@@ -787,6 +791,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { s8 - s31} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #2					// ldc = ldc * 4
 	str	r3, LDC

kernel/arm/zdot_vfp.S

@@ -187,13 +187,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r4, fp, #128
 	vstm	r4, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0                                          // clear floating point register
+        vmov    s0, r4
+        vcvt.f64.f32    d0, s0
+        vcvt.f64.f32    d1, s0
+        vcvt.f64.f32    d2, s0
+        vcvt.f64.f32    d3, s0
+
 	mov	Y, OLD_Y
 	ldr	INC_Y, OLD_INC_Y
 
-	vsub.f64                d0 , d0 , d0
-	vsub.f64                d1 , d1 , d1
-	vsub.f64                d2 , d2 , d2
-	vsub.f64                d3 , d3 , d3
 
 	cmp	N, #0
 	ble	zdot_kernel_L999

kernel/arm/zgemm_kernel_2x2_vfp.S

@@ -57,6 +57,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -131,7 +135,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -383,7 +387,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d12, d8
 	vmov.f64		d13, d8
@@ -557,7 +561,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -724,7 +728,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 
 .endm
@@ -869,6 +873,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #4					// ldc = ldc * 8 * 2
 	str	r3, LDC

kernel/arm/zgemm_kernel_2x2_vfpv3.S

@@ -73,6 +73,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-240]
+#define FP_ZERO_0 [fp, # -240]
+#define FP_ZERO_1 [fp, # -236]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -147,7 +151,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -404,7 +408,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -586,7 +590,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -766,7 +770,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16, FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d24, d16
 	vmov.f64		d25, d16
@@ -915,6 +919,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #4					// ldc = ldc * 8 * 2
 	str	r3, LDC

kernel/arm/zgemv_n_vfp.S

@@ -59,6 +59,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
+
 #define ALPHA_I [fp, #-236]
 #define ALPHA_R [fp, #-244]
 
@@ -117,7 +122,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .macro INIT_F4
 
 	pld	[ YO, #Y_PRE ]
-        vsub.f64                d8 , d8 , d8
+	fldd			d8, FP_ZERO
         vmov.f64                d9 , d8
         vmov.f64                d10, d8
         vmov.f64                d11, d8
@@ -222,7 +227,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-        vsub.f64                d8 , d8 , d8
+	fldd			d8, FP_ZERO
         vmov.f64                d9 , d8
 
 .endm
@@ -269,7 +274,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S4
 
-        vsub.f64                d8 , d8 , d8
+	fldd			d8, FP_ZERO
         vmov.f64                d9 , d8
         vmov.f64                d10, d8
         vmov.f64                d11, d8
@@ -386,7 +391,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-        vsub.f64                d8 , d8 , d8
+	fldd			d8, FP_ZERO
         vmov.f64                d9 , d8
 
 .endm
@@ -450,6 +455,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	OLD_M, #0
 	ble	zgemvn_kernel_L999
 

kernel/arm/zgemv_t_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define I	r12
 
+#define FP_ZERO [fp, #-228]
+#define FP_ZERO_0 [fp, #-228]
+#define FP_ZERO_1 [fp, #-224]
+
 #define N	[fp, #-252 ]
 #define A	[fp, #-256 ]
 
@@ -117,10 +121,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F2
 
-	vsub.f64	d12, d12, d12
-	vsub.f64	d13, d13, d13
-	vsub.f64	d14, d14, d14
-	vsub.f64	d15, d15, d15
+	fldd		d12, FP_ZERO
+	vmov.f64	d13, d12
+	vmov.f64	d14, d12
+	vmov.f64	d15, d12
 
 .endm
 
@@ -173,8 +177,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_F1
 
-	vsub.f64	d12, d12, d12
-	vsub.f64	d13, d13, d13
+	fldd		d12, FP_ZERO
+	vmov.f64	d13, d12
 
 .endm
 
@@ -216,10 +220,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S2
 
-	vsub.f64	d12, d12, d12
-	vsub.f64	d13, d13, d13
-	vsub.f64	d14, d14, d14
-	vsub.f64	d15, d15, d15
+	fldd		d12, FP_ZERO
+	vmov.f64	d13, d12
+	vmov.f64	d14, d12
+	vmov.f64	d15, d12
 
 .endm
 
@@ -282,8 +286,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT_S1
 
-	vsub.f64	d12, d12, d12
-	vsub.f64	d13, d13, d13
+	fldd		d12, FP_ZERO
+	vmov.f64	d13, d12
 
 .endm
 
@@ -346,6 +350,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         vstm    r12, { s8 - s15 }                                 // store floating point registers
 #endif
 
+        movs    r12, #0
+        str     r12, FP_ZERO
+        str     r12, FP_ZERO_1
+
 	cmp	M, #0
 	ble	zgemvt_kernel_L999
 

kernel/arm/ztrmm_kernel_2x2_vfp.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-232]
+#define FP_ZERO_0 [fp, #-232]
+#define FP_ZERO_1 [fp, #-228]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -140,7 +144,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -356,10 +360,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	fldd		d0, ALPHA_R
 	fldd		d1, ALPHA_I
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
-	vsub.f64	d6, d6 , d6
-	vsub.f64	d7, d7 , d7
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
+	vmov.f64	d6 , d4
+	vmov.f64	d7 , d4
 
 	FMAC_R1 d4 , d0 , d8
 	FMAC_I1 d5 , d0 , d9
@@ -373,10 +377,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	fstmiad CO1, { d4 - d7 }
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
-	vsub.f64	d6, d6 , d6
-	vsub.f64	d7, d7 , d7
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
+	vmov.f64	d6 , d4
+	vmov.f64	d7 , d4
 
 	FMAC_R1 d4 , d0 , d12
 	FMAC_I1 d5 , d0 , d13
@@ -398,7 +402,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d12, d8
 	vmov.f64		d13, d8
@@ -545,8 +549,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	fldd		d0, ALPHA_R
 	fldd		d1, ALPHA_I
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
 
 	FMAC_R1 d4 , d0 , d8
 	FMAC_I1 d5 , d0 , d9
@@ -555,8 +559,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	fstmiad CO1, { d4 - d5 }
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
 
 	FMAC_R1 d4 , d0 , d12
 	FMAC_I1 d5 , d0 , d13
@@ -574,7 +578,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 	vmov.f64		d10, d8
 	vmov.f64		d11, d8
@@ -718,10 +722,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	fldd		d0, ALPHA_R
 	fldd		d1, ALPHA_I
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
-	vsub.f64	d6, d6 , d6
-	vsub.f64	d7, d7 , d7
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
+	vmov.f64	d6 , d4
+	vmov.f64	d7 , d4
 
 	FMAC_R1 d4 , d0 , d8
 	FMAC_I1 d5 , d0 , d9
@@ -744,7 +748,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d8 , d8 , d8
+	fldd			d8 , FP_ZERO
 	vmov.f64		d9 , d8
 
 .endm
@@ -850,8 +854,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	fldd		d0, ALPHA_R
 	fldd		d1, ALPHA_I
 
-	vsub.f64	d4, d4 , d4
-	vsub.f64	d5, d5 , d5
+	fldd		d4 , FP_ZERO
+	vmov.f64	d5 , d4
 
 	FMAC_R1 d4 , d0 , d8
 	FMAC_I1 d5 , d0 , d9
@@ -888,6 +892,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #4					// ldc = ldc * 8 * 2
 	str	r3, LDC

kernel/arm/ztrmm_kernel_2x2_vfpv3.S

@@ -59,6 +59,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define N	[fp, #-260 ]
 #define K	[fp, #-264 ]
 
+#define FP_ZERO [fp, #-236]
+#define FP_ZERO_0 [fp, #-236]
+#define FP_ZERO_1 [fp, #-232]
+
 #define ALPHA_I	[fp, #-272]
 #define ALPHA_R	[fp, #-280]
 
@@ -134,7 +138,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16 , FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -388,7 +392,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x2
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16 , FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d20, d16
 	vmov.f64		d21, d16
@@ -566,7 +570,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT2x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16 , FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d18, d16
 	vmov.f64		d19, d16
@@ -743,7 +747,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro INIT1x1
 
-	vsub.f64		d16 , d16 , d16
+	fldd			d16 , FP_ZERO
 	vmov.f64		d17, d16
 	vmov.f64		d24, d16
 	vmov.f64		d25, d16
@@ -889,6 +893,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	sub	r3, fp, #128
 	vstm	r3, { d8 - d15} 				// store floating point registers
 
+        movs    r4, #0
+        str     r4, FP_ZERO
+        str     r4, FP_ZERO_1
+
 	ldr	r3, OLD_LDC
 	lsl	r3, r3, #4					// ldc = ldc * 8 * 2
 	str	r3, LDC

kernel/generic/zgemmkernel_2x2.c

@@ -797,7 +797,7 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alphar,FLOAT alphai,FLOAT* b
                   res1 = res1-load0*load3;
 #endif
 #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)
-				  load0 = ptrba[2*0+0];
+                  load0 = ptrba[2*0+0];
                   load1 = ptrbb[2*0+0];
                   res0 = res0+load0*load1;
                   load2 = ptrba[2*0+1];

kernel/power/KERNEL

@@ -1,57 +1,3 @@
-SGEMM_BETA = gemm_beta.S
-DGEMM_BETA = gemm_beta.S
-CGEMM_BETA = zgemm_beta.S
-ZGEMM_BETA = zgemm_beta.S
-
-
-ifndef SSYMV_U_KERNEL
-SSYMV_U_KERNEL =  symv_U.S
-endif
-
-ifndef SSYMV_L_KERNEL
-SSYMV_L_KERNEL =  symv_L.S
-endif
-
-ifndef DSYMV_U_KERNEL
-DSYMV_U_KERNEL =  symv_U.S
-endif
-
-ifndef DSYMV_L_KERNEL
-DSYMV_L_KERNEL =  symv_L.S
-endif
-
-ifndef CSYMV_U_KERNEL
-CSYMV_U_KERNEL =  zsymv_U.S
-endif
-
-ifndef CSYMV_L_KERNEL
-CSYMV_L_KERNEL =  zsymv_L.S
-endif
-
-ifndef ZSYMV_U_KERNEL
-ZSYMV_U_KERNEL =  zsymv_U.S
-endif
-
-ifndef ZSYMV_L_KERNEL
-ZSYMV_L_KERNEL =  zsymv_L.S
-endif
-
-ifndef CHEMV_U_KERNEL
-CHEMV_U_KERNEL =  zsymv_U.S
-endif
-
-ifndef CHEMV_L_KERNEL
-CHEMV_L_KERNEL =  zsymv_L.S
-endif
-
-ifndef ZHEMV_U_KERNEL
-ZHEMV_U_KERNEL =  zsymv_U.S
-endif
-
-ifndef ZHEMV_L_KERNEL
-ZHEMV_L_KERNEL =  zsymv_L.S
-endif
-
 ifndef STRSMKERNEL_LN
 STRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
 endif
@@ -84,3 +30,19 @@ ifndef CTRSMKERNEL_RT
 CTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
 endif
 
+ifndef SGEMM_BETA
+SGEMM_BETA = gemm_beta.S
+endif
+
+ifndef DGEMM_BETA
+DGEMM_BETA = gemm_beta.S
+endif
+
+ifndef CGEMM_BETA
+CGEMM_BETA = zgemm_beta.S
+endif
+
+ifndef ZGEMM_BETA
+ZGEMM_BETA = zgemm_beta.S
+endif
+

kernel/power/KERNEL.POWER8

@@ -0,0 +1,175 @@
+#SGEMM_BETA = ../generic/gemm_beta.c
+#DGEMM_BETA = ../generic/gemm_beta.c
+#CGEMM_BETA = ../generic/zgemm_beta.c
+#ZGEMM_BETA = ../generic/zgemm_beta.c
+
+STRMMKERNEL	= gemm_kernel_power6.S
+DTRMMKERNEL	= dtrmm_kernel_16x4_power8.S
+CTRMMKERNEL	= ../generic/ztrmmkernel_2x2.c
+ZTRMMKERNEL	= ztrmm_kernel_8x2_power8.S
+
+SGEMMKERNEL    =  gemm_kernel_power6.S
+SGEMMONCOPY    =  ../generic/gemm_ncopy_4.c
+SGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c
+SGEMMONCOPYOBJ =  sgemm_oncopy.o
+SGEMMOTCOPYOBJ =  sgemm_otcopy.o
+
+DGEMMKERNEL    =  dgemm_kernel_16x4_power8.S
+DGEMMINCOPY    = ../generic/gemm_ncopy_16.c
+DGEMMITCOPY    = ../generic/gemm_tcopy_16.c
+DGEMMONCOPY    =  gemm_ncopy_4.S
+DGEMMOTCOPY    =  gemm_tcopy_4.S
+DGEMMINCOPYOBJ = dgemm_incopy.o
+DGEMMITCOPYOBJ = dgemm_itcopy.o
+DGEMMONCOPYOBJ = dgemm_oncopy.o
+DGEMMOTCOPYOBJ = dgemm_otcopy.o
+
+CGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
+CGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
+CGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
+CGEMMONCOPYOBJ =  cgemm_oncopy.o
+CGEMMOTCOPYOBJ =  cgemm_otcopy.o
+
+ZGEMMKERNEL    = zgemm_kernel_8x2_power8.S
+ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
+ZGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
+ZGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
+ZGEMMITCOPY    = ../generic/zgemm_tcopy_8.c
+ZGEMMONCOPYOBJ =  zgemm_oncopy.o
+ZGEMMOTCOPYOBJ =  zgemm_otcopy.o
+ZGEMMINCOPYOBJ =  zgemm_incopy.o
+ZGEMMITCOPYOBJ =  zgemm_itcopy.o
+
+STRSMKERNEL_LN	=  ../generic/trsm_kernel_LN.c
+STRSMKERNEL_LT	=  ../generic/trsm_kernel_LT.c
+STRSMKERNEL_RN	=  ../generic/trsm_kernel_RN.c
+STRSMKERNEL_RT	=  ../generic/trsm_kernel_RT.c
+
+DTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+DTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
+DTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+DTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+CTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+CTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
+CTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+CTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+ZTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
+ZTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
+ZTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+ZTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
+
+#Todo: CGEMM3MKERNEL should be 4x4 blocksizes.
+#CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_sse3.S
+#ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_sse3.S
+
+#Pure C for other kernels
+#SAMAXKERNEL  = ../arm/amax.c
+#DAMAXKERNEL  = ../arm/amax.c
+#CAMAXKERNEL  = ../arm/zamax.c
+#ZAMAXKERNEL  = ../arm/zamax.c
+#
+#SAMINKERNEL  = ../arm/amin.c
+#DAMINKERNEL  = ../arm/amin.c
+#CAMINKERNEL  = ../arm/zamin.c
+#ZAMINKERNEL  = ../arm/zamin.c
+#
+#SMAXKERNEL   = ../arm/max.c
+#DMAXKERNEL   = ../arm/max.c
+#
+#SMINKERNEL   = ../arm/min.c
+#DMINKERNEL   = ../arm/min.c
+#
+#ISAMAXKERNEL = ../arm/iamax.c
+#IDAMAXKERNEL = ../arm/iamax.c
+#ICAMAXKERNEL = ../arm/izamax.c
+#IZAMAXKERNEL = ../arm/izamax.c
+#
+#ISAMINKERNEL = ../arm/iamin.c
+#IDAMINKERNEL = ../arm/iamin.c
+#ICAMINKERNEL = ../arm/izamin.c
+#IZAMINKERNEL = ../arm/izamin.c
+#
+#ISMAXKERNEL  = ../arm/imax.c
+#IDMAXKERNEL  = ../arm/imax.c
+#
+#ISMINKERNEL  = ../arm/imin.c
+#IDMINKERNEL  = ../arm/imin.c
+#
+#SASUMKERNEL  = ../arm/asum.c
+#DASUMKERNEL  = ../arm/asum.c
+#CASUMKERNEL  = ../arm/zasum.c
+#ZASUMKERNEL  = ../arm/zasum.c
+#
+#SAXPYKERNEL  = ../arm/axpy.c
+#DAXPYKERNEL  = ../arm/axpy.c
+#CAXPYKERNEL  = ../arm/zaxpy.c
+#ZAXPYKERNEL  = ../arm/zaxpy.c
+#
+#SCOPYKERNEL  = ../arm/copy.c
+#DCOPYKERNEL  = ../arm/copy.c
+#CCOPYKERNEL  = ../arm/zcopy.c
+#ZCOPYKERNEL  = ../arm/zcopy.c
+#
+#SDOTKERNEL   = ../arm/dot.c
+#DDOTKERNEL   = ../arm/dot.c
+#CDOTKERNEL   = ../arm/zdot.c
+#ZDOTKERNEL   = ../arm/zdot.c
+#
+#SNRM2KERNEL  = ../arm/nrm2.c
+#DNRM2KERNEL  = ../arm/nrm2.c
+#CNRM2KERNEL  = ../arm/znrm2.c
+#ZNRM2KERNEL  = ../arm/znrm2.c
+#
+#SROTKERNEL   = ../arm/rot.c
+#DROTKERNEL   = ../arm/rot.c
+#CROTKERNEL   = ../arm/zrot.c
+#ZROTKERNEL   = ../arm/zrot.c
+#
+#SSCALKERNEL  = ../arm/scal.c
+#DSCALKERNEL  = ../arm/scal.c
+#CSCALKERNEL  = ../arm/zscal.c
+#ZSCALKERNEL  = ../arm/zscal.c
+#
+#SSWAPKERNEL  = ../arm/swap.c
+#DSWAPKERNEL  = ../arm/swap.c
+#CSWAPKERNEL  = ../arm/zswap.c
+#ZSWAPKERNEL  = ../arm/zswap.c
+#
+
+#SGEMVNKERNEL = ../arm/gemv_n.c
+#DGEMVNKERNEL = ../arm/gemv_n.c
+#CGEMVNKERNEL = ../arm/zgemv_n.c
+#ZGEMVNKERNEL = ../arm/zgemv_n.c
+#
+#SGEMVTKERNEL = ../arm/gemv_t.c
+#DGEMVTKERNEL = ../arm/gemv_t.c
+#CGEMVTKERNEL = ../arm/zgemv_t.c
+#ZGEMVTKERNEL = ../arm/zgemv_t.c
+
+
+#SSYMV_U_KERNEL =  ../generic/symv_k.c
+#SSYMV_L_KERNEL =  ../generic/symv_k.c
+#DSYMV_U_KERNEL =  ../generic/symv_k.c
+#DSYMV_L_KERNEL =  ../generic/symv_k.c
+#QSYMV_U_KERNEL =  ../generic/symv_k.c
+#QSYMV_L_KERNEL =  ../generic/symv_k.c
+#CSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#CSYMV_L_KERNEL =  ../generic/zsymv_k.c
+#ZSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#ZSYMV_L_KERNEL =  ../generic/zsymv_k.c
+#XSYMV_U_KERNEL =  ../generic/zsymv_k.c
+#XSYMV_L_KERNEL =  ../generic/zsymv_k.c
+
+#ZHEMV_U_KERNEL =  ../generic/zhemv_k.c
+#ZHEMV_L_KERNEL =  ../generic/zhemv_k.c
+
+LSAME_KERNEL = ../generic/lsame.c
+SCABS_KERNEL	= ../generic/cabs.c
+DCABS_KERNEL	= ../generic/cabs.c
+QCABS_KERNEL	= ../generic/cabs.c
+
+#Dump kernel
+CGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c
+ZGEMM3MKERNEL    = ../generic/zgemm3mkernel_dump.c

kernel/power/def_vsx.h

@@ -0,0 +1,64 @@
+#define vs0 0
+#define vs1 1
+#define vs2 2
+#define vs3 3
+#define vs4 4
+#define vs5 5
+#define vs6 6
+#define vs7 7
+#define vs8 8
+#define vs9 9
+#define vs10 10
+#define vs11 11
+#define vs12 12
+#define vs13 13
+#define vs14 14
+#define vs15 15
+#define vs16 16
+#define vs17 17
+#define vs18 18
+#define vs19 19
+#define vs20 20
+#define vs21 21
+#define vs22 22
+#define vs23 23
+#define vs24 24
+#define vs25 25
+#define vs26 26
+#define vs27 27
+#define vs28 28
+#define vs29 29
+#define vs30 30
+#define vs31 31
+#define vs32 32
+#define vs33 33
+#define vs34 34
+#define vs35 35
+#define vs36 36
+#define vs37 37
+#define vs38 38
+#define vs39 39
+#define vs40 40
+#define vs41 41
+#define vs42 42
+#define vs43 43
+#define vs44 44
+#define vs45 45
+#define vs46 46
+#define vs47 47
+#define vs48 48
+#define vs49 49
+#define vs50 50
+#define vs51 51
+#define vs52 52
+#define vs53 53
+#define vs54 54
+#define vs55 55
+#define vs56 56
+#define vs57 57
+#define vs58 58
+#define vs59 59
+#define vs60 60
+#define vs61 61
+#define vs62 62
+#define vs63 63

kernel/power/dgemm_kernel_16x4_power8.S

@@ -0,0 +1,348 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 320
+#define ALPHA_SP   296(SP)
+#define FZERO	304(SP)
+#else
+#define STACKSIZE 240
+#define ALPHA_SP   224(SP)
+#define FZERO	232(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r7
+#define OFFSET	r6
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#define alpha_r vs18
+
+#define o0	0
+
+#define o8	r15
+#define o24	r16
+#define ALPHA	r17
+#define L	r18
+#define T1	r19
+#define KK	r20
+#define BB	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO 	r26
+#define o16	r27
+#define	o32	r28
+#define	o48	r29
+
+#define PRE	r30
+#define T2	r31
+
+#include "dgemm_macros_16x4_power8.S"
+
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+#endif
+
+	stfd	f1,  ALPHA_SP
+	stw	r0,  FZERO
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+	slwi	LDC, LDC, BASE_SHIFT
+
+#if defined(TRMMKERNEL)
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#endif
+
+
+	cmpwi	cr0, M, 0
+	ble	.L999_H1
+	cmpwi	cr0, N, 0
+	ble	.L999_H1
+	cmpwi	cr0, K, 0
+	ble	.L999_H1
+
+#ifdef __64BIT__
+	addi	ALPHA, SP, 296
+#else
+	addi	ALPHA, SP, 224
+#endif
+
+	li	PRE, 256 
+	li	o8 , 8
+	li	o16, 16
+	li	o24, 24
+	li	o32, 32
+	li	o48, 48
+
+	lxvdsx	alpha_r, 0, ALPHA
+
+#include "dgemm_logic_16x4_power8.S"
+
+.L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/dtrmm_kernel_16x4_power8.S

@@ -0,0 +1,362 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 320
+#define ALPHA_SP   296(SP)
+#define FZERO	304(SP)
+#else
+#define STACKSIZE 240
+#define ALPHA_SP   224(SP)
+#define FZERO	232(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r7
+#define OFFSET	r6
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#define alpha_r vs18
+
+#define o0	0
+
+#define K1	r13
+#define KKK	r14
+#define o8	r15
+#define o24	r16
+#define ALPHA	r17
+#define L	r18
+#define T1	r19
+#define KK	r20
+#define BB	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO 	r26
+#define o16	r27
+#define	o32	r28
+#define	o48	r29
+
+#define PRE	r30
+#define T2	r31
+
+#include "dgemm_macros_16x4_power8.S"
+
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+	std	r13,  288(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+	stw	r13,  216(SP)
+#endif
+
+	stfd	f1,  ALPHA_SP
+	stw	r0,  FZERO
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+	slwi	LDC, LDC, BASE_SHIFT
+
+#if defined(TRMMKERNEL)
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#endif
+
+	mr	KK, OFFSET
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        neg     KK, KK
+#endif
+
+	cmpwi	cr0, M, 0
+	ble	.L999_H1
+	cmpwi	cr0, N, 0
+	ble	.L999_H1
+	cmpwi	cr0, K, 0
+	ble	.L999_H1
+
+#ifdef __64BIT__
+	addi	ALPHA, SP, 296
+#else
+	addi	ALPHA, SP, 224
+#endif
+
+	li	PRE, 256 
+	li	o8 , 8
+	li	o16, 16
+	li	o24, 24
+	li	o32, 32
+	li	o48, 48
+
+	lxvdsx	alpha_r, 0, ALPHA
+
+#include "dtrmm_logic_16x4_power8.S"
+
+.L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+	ld	r13,  288(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+	lwz	r13,  216(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/gemm_ncopy_4.S

@@ -107,6 +107,11 @@
 #ifdef PPCG4
 #define PREFETCHSIZE   16
 #define PREFETCHWSIZE  72
+#endif
+
+#ifdef POWER8
+#define PREFETCHSIZE   16
+#define PREFETCHWSIZE  72
 #endif
 
 	PROLOGUE
@@ -193,7 +198,7 @@ LL(12):
 	STFD	c12,  14 * SIZE(B)
 	STFD	c16,  15 * SIZE(B)
 
-#ifdef POWER6
+#if defined(POWER6) || defined(POWER8)
 	dcbtst	PREA, AO1
 	dcbtst	PREA, AO2
 	dcbtst	PREA, AO3

kernel/power/gemm_tcopy_4.S

@@ -111,6 +111,11 @@
 #ifdef PPCG4
 #define PREFETCHSIZE   16
 #define PREFETCHWSIZE  48
+#endif
+
+#ifdef POWER8
+#define PREFETCHSIZE   16
+#define PREFETCHWSIZE  48
 #endif
 
 	PROLOGUE
@@ -224,7 +229,7 @@ LL(12):
 	STFD	c15,  14 * SIZE(B1)
 	STFD	c16,  15 * SIZE(B1)
 
-#ifdef POWER6
+#if defined(POWER6) || defined(POWER8)
 	dcbtst	PREA, AO1
 	dcbtst	PREA, AO2
 	dcbtst	PREA, AO3

kernel/power/gemv_n.S

@@ -174,6 +174,12 @@
 #define PREFETCHSIZE_C  40
 #endif
 
+#ifdef POWER8
+#define PREFETCHSIZE_A  96
+#define PREFETCHSIZE_C  40
+#endif
+
+
 #ifndef NEEDPARAM
 
 #ifndef __64BIT__

kernel/power/gemv_t.S

@@ -139,6 +139,11 @@
 #define PREFETCHSIZE_C   8
 #endif
 
+#ifdef POWER8
+#define PREFETCHSIZE_A  96
+#define PREFETCHSIZE_C   8
+#endif
+
 #define y01 f0
 #define y02 f1
 #define y03 f2

kernel/power/zgemm_kernel_8x2_power8.S

@@ -0,0 +1,367 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 320
+#define ALPHA_R_SP 296(SP)
+#define ALPHA_I_SP 304(SP)
+#define FZERO	312(SP)
+#else
+#define STACKSIZE 256
+#define ALPHA_R_SP 224(SP)
+#define ALPHA_I_SP 232(SP)
+#define FZERO	240(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r10
+#define	B	r6
+#define	C	r7
+#define	LDC	r8
+#define OFFSET	r9
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#define o0	0
+#define alpha_r vs30
+#define alpha_i vs31
+
+#define L	r15
+#define ALPHA	r16
+#define o24	r17
+#define T2	r19
+#define KK	r20
+#define	o8	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define o16	r27
+#define	o32	r28
+#define o48	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+#endif
+
+	stfd	f1,  ALPHA_R_SP
+	stfd	f2,  ALPHA_I_SP
+	stw	r0,  FZERO
+
+#ifdef linux
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)
+	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)
+	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)
+#else
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+
+#ifdef TRMMKERNEL
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+	neg	KK, OFFSET
+#endif
+#endif
+
+#include "zgemm_macros_8x2_power8.S"
+
+	cmpwi	cr0, M, 0
+	ble	.L999
+	cmpwi	cr0, N, 0
+	ble	.L999
+	cmpwi	cr0, K, 0
+	ble	.L999
+
+	slwi	LDC, LDC, ZBASE_SHIFT
+	li	PRE, 256 
+	li	o8  , 8
+	li	o16 , 16
+	li	o24 , 24
+	li	o32 , 32
+	li	o48 , 48
+
+#ifdef __64BIT__
+	addi	ALPHA, SP, 296
+#else
+	addi	ALPHA, SP, 224
+#endif
+
+	lxvdsx	alpha_r, 0, ALPHA
+	lxvdsx	alpha_i, o8, ALPHA
+
+	.align 5
+
+#include "zgemm_logic_8x2_power8.S"
+
+.L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/zgemm_logic_8x2_power8.S

@@ -0,0 +1,901 @@
+	srawi.		J,	N,	1
+	ble		.LZGEMM_L2_END
+
+.LZGEMM_L2_BEGIN:
+
+	mr		CO,	C
+	mr		AO,	A
+	slwi		T1,	LDC	,	1
+	add		C,	C,	T1
+	srawi.		I,	M,	3
+	ble		.LZGEMM_L2x8_END
+
+.LZGEMM_L2x8_BEGIN:
+
+
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L2x8_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L2x8_SUB4
+
+.LZGEMM_L2x8_LOOP_START:
+
+	dcbt		AO,	PRE
+	LOAD2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_I1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L2x8_LOOP_END
+
+	.align 5
+
+.LZGEMM_L2x8_LOOP:
+
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x8_LOOP
+
+.LZGEMM_L2x8_LOOP_END:
+
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	dcbt		AO,	PRE
+	KERNEL2x8_2
+	dcbt		AO,	PRE
+	KERNEL2x8_1
+	KERNEL2x8_E2
+
+	b		.LZGEMM_L2x8_SUB1
+
+.LZGEMM_L2x8_SUB4:
+
+	dcbt		AO,	PRE
+	KERNEL2x8_SUBI1
+	dcbt		AO,	PRE
+	KERNEL2x8_SUB1
+	dcbt		AO,	PRE
+	KERNEL2x8_SUB1
+	dcbt		AO,	PRE
+	KERNEL2x8_SUB1
+
+	KERNEL2x8_SUB1
+	KERNEL2x8_SUB1
+	KERNEL2x8_SUB1
+	KERNEL2x8_SUB1
+
+	b		.LZGEMM_L2x8_SUB1
+
+.LZGEMM_L2x8_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL2x8_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L2x8_SAVE
+	b		.LZGEMM_L2x8_SUB2
+
+.LZGEMM_L2x8_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L2x8_SAVE
+
+.LZGEMM_L2x8_SUB2:
+
+	KERNEL2x8_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x8_SUB2
+
+.LZGEMM_L2x8_SAVE:
+
+	SAVE2x8
+
+	addic.		I,	I,	-1
+	bgt		.LZGEMM_L2x8_BEGIN
+
+.LZGEMM_L2x8_END:
+
+.LZGEMM_L2x4_BEGIN:
+
+	andi.		T2,	M,	7
+	ble		.LZGEMM_L2x1_END
+
+	andi.		T1,	M,	4
+	ble		.LZGEMM_L2x4_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L2x4_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L2x4_SUB4
+
+.LZGEMM_L2x4_LOOP_START:
+
+	LOAD2x4_1
+	KERNEL2x4_I1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_2
+
+	KERNEL2x4_1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L2x4_LOOP_END
+
+	.align 5
+
+.LZGEMM_L2x4_LOOP:
+
+	KERNEL2x4_1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_2
+
+	KERNEL2x4_1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x4_LOOP
+
+.LZGEMM_L2x4_LOOP_END:
+
+	KERNEL2x4_1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_2
+
+	KERNEL2x4_1
+	KERNEL2x4_2
+	KERNEL2x4_1
+	KERNEL2x4_E2
+
+	b		.LZGEMM_L2x4_SUB1
+
+.LZGEMM_L2x4_SUB4:
+
+	KERNEL2x4_SUBI1
+	KERNEL2x4_SUB1
+	KERNEL2x4_SUB1
+	KERNEL2x4_SUB1
+
+	KERNEL2x4_SUB1
+	KERNEL2x4_SUB1
+	KERNEL2x4_SUB1
+	KERNEL2x4_SUB1
+
+	b		.LZGEMM_L2x4_SUB1
+
+.LZGEMM_L2x4_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL2x4_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L2x4_SAVE
+	b		.LZGEMM_L2x4_SUB2
+
+.LZGEMM_L2x4_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L2x4_SAVE
+
+.LZGEMM_L2x4_SUB2:
+
+	KERNEL2x4_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x4_SUB2
+
+.LZGEMM_L2x4_SAVE:
+
+	SAVE2x4
+
+.LZGEMM_L2x4_END:
+
+.LZGEMM_L2x2_BEGIN:
+
+
+	andi.		T1,	M,	2
+	ble		.LZGEMM_L2x2_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L2x2_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L2x2_SUB4
+
+.LZGEMM_L2x2_LOOP_START:
+
+	LOAD2x2_1
+	KERNEL2x2_I1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_2
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L2x2_LOOP_END
+
+	.align 5
+
+.LZGEMM_L2x2_LOOP:
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_2
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x2_LOOP
+
+.LZGEMM_L2x2_LOOP_END:
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_2
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_1
+	KERNEL2x2_E2
+
+	b		.LZGEMM_L2x2_SUB1
+
+.LZGEMM_L2x2_SUB4:
+
+	KERNEL2x2_SUBI1
+	KERNEL2x2_SUB1
+	KERNEL2x2_SUB1
+	KERNEL2x2_SUB1
+
+	KERNEL2x2_SUB1
+	KERNEL2x2_SUB1
+	KERNEL2x2_SUB1
+	KERNEL2x2_SUB1
+
+	b		.LZGEMM_L2x2_SUB1
+
+.LZGEMM_L2x2_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL2x2_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L2x2_SAVE
+	b		.LZGEMM_L2x2_SUB2
+
+.LZGEMM_L2x2_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L2x2_SAVE
+
+.LZGEMM_L2x2_SUB2:
+
+	KERNEL2x2_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x2_SUB2
+
+.LZGEMM_L2x2_SAVE:
+
+	SAVE2x2
+
+.LZGEMM_L2x2_END:
+
+.LZGEMM_L2x1_BEGIN:
+
+
+	andi.		T1,	M,	1
+	ble		.LZGEMM_L2x1_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L2x1_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L2x1_SUB4
+
+.LZGEMM_L2x1_LOOP_START:
+
+	LOAD2x1_1
+	KERNEL2x1_I1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_2
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L2x1_LOOP_END
+
+	.align 5
+
+.LZGEMM_L2x1_LOOP:
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_2
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x1_LOOP
+
+.LZGEMM_L2x1_LOOP_END:
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_2
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_1
+	KERNEL2x1_E2
+
+	b		.LZGEMM_L2x1_SUB1
+
+.LZGEMM_L2x1_SUB4:
+
+	KERNEL2x1_SUBI1
+	KERNEL2x1_SUB1
+	KERNEL2x1_SUB1
+	KERNEL2x1_SUB1
+
+	KERNEL2x1_SUB1
+	KERNEL2x1_SUB1
+	KERNEL2x1_SUB1
+	KERNEL2x1_SUB1
+
+	b		.LZGEMM_L2x1_SUB1
+
+.LZGEMM_L2x1_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL2x1_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L2x1_SAVE
+	b		.LZGEMM_L2x1_SUB2
+
+.LZGEMM_L2x1_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L2x1_SAVE
+
+.LZGEMM_L2x1_SUB2:
+
+	KERNEL2x1_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L2x1_SUB2
+
+.LZGEMM_L2x1_SAVE:
+
+	SAVE2x1
+
+.LZGEMM_L2x1_END:
+
+	slwi		T1,	K,	5
+	add		B,	B,	T1
+
+	addic.		J,	J,	-1
+	bgt		.LZGEMM_L2_BEGIN
+
+	andi.		T2,	N,	1
+	ble		.L999
+
+.LZGEMM_L2_END:
+
+	b		.LZGEMM_L1_BEGIN
+
+.L999_H1:
+
+	b		.L999
+
+.LZGEMM_L1_BEGIN:
+
+	andi.		T1,	N,	1
+	ble		.LZGEMM_L1_END
+	mr		CO,	C
+	mr		AO,	A
+	srawi.		I,	M,	3
+	ble		.LZGEMM_L1x8_END
+
+.LZGEMM_L1x8_BEGIN:
+
+
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L1x8_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L1x8_SUB4
+
+.LZGEMM_L1x8_LOOP_START:
+
+	dcbt		AO,	PRE
+	LOAD1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_I1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L1x8_LOOP_END
+
+	.align 5
+
+.LZGEMM_L1x8_LOOP:
+
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x8_LOOP
+
+.LZGEMM_L1x8_LOOP_END:
+
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	dcbt		AO,	PRE
+	KERNEL1x8_2
+	dcbt		AO,	PRE
+	KERNEL1x8_1
+	KERNEL1x8_E2
+
+	b		.LZGEMM_L1x8_SUB1
+
+.LZGEMM_L1x8_SUB4:
+
+	dcbt		AO,	PRE
+	KERNEL1x8_SUBI1
+	dcbt		AO,	PRE
+	KERNEL1x8_SUB1
+	dcbt		AO,	PRE
+	KERNEL1x8_SUB1
+	dcbt		AO,	PRE
+	KERNEL1x8_SUB1
+
+	KERNEL1x8_SUB1
+	KERNEL1x8_SUB1
+	KERNEL1x8_SUB1
+	KERNEL1x8_SUB1
+
+	b		.LZGEMM_L1x8_SUB1
+
+.LZGEMM_L1x8_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL1x8_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L1x8_SAVE
+	b		.LZGEMM_L1x8_SUB2
+
+.LZGEMM_L1x8_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L1x8_SAVE
+
+.LZGEMM_L1x8_SUB2:
+
+	KERNEL1x8_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x8_SUB2
+
+.LZGEMM_L1x8_SAVE:
+
+	SAVE1x8
+
+	addic.		I,	I,	-1
+	bgt		.LZGEMM_L1x8_BEGIN
+
+.LZGEMM_L1x8_END:
+
+.LZGEMM_L1x4_BEGIN:
+
+	andi.		T2,	M,	7
+	ble		.LZGEMM_L1x1_END
+
+	andi.		T1,	M,	4
+	ble		.LZGEMM_L1x4_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L1x4_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L1x4_SUB4
+
+.LZGEMM_L1x4_LOOP_START:
+
+	LOAD1x4_1
+	KERNEL1x4_I1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_2
+
+	KERNEL1x4_1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L1x4_LOOP_END
+
+	.align 5
+
+.LZGEMM_L1x4_LOOP:
+
+	KERNEL1x4_1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_2
+
+	KERNEL1x4_1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x4_LOOP
+
+.LZGEMM_L1x4_LOOP_END:
+
+	KERNEL1x4_1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_2
+
+	KERNEL1x4_1
+	KERNEL1x4_2
+	KERNEL1x4_1
+	KERNEL1x4_E2
+
+	b		.LZGEMM_L1x4_SUB1
+
+.LZGEMM_L1x4_SUB4:
+
+	KERNEL1x4_SUBI1
+	KERNEL1x4_SUB1
+	KERNEL1x4_SUB1
+	KERNEL1x4_SUB1
+
+	KERNEL1x4_SUB1
+	KERNEL1x4_SUB1
+	KERNEL1x4_SUB1
+	KERNEL1x4_SUB1
+
+	b		.LZGEMM_L1x4_SUB1
+
+.LZGEMM_L1x4_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL1x4_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L1x4_SAVE
+	b		.LZGEMM_L1x4_SUB2
+
+.LZGEMM_L1x4_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L1x4_SAVE
+
+.LZGEMM_L1x4_SUB2:
+
+	KERNEL1x4_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x4_SUB2
+
+.LZGEMM_L1x4_SAVE:
+
+	SAVE1x4
+
+.LZGEMM_L1x4_END:
+
+.LZGEMM_L1x2_BEGIN:
+
+
+	andi.		T1,	M,	2
+	ble		.LZGEMM_L1x2_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L1x2_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L1x2_SUB4
+
+.LZGEMM_L1x2_LOOP_START:
+
+	LOAD1x2_1
+	KERNEL1x2_I1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_2
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L1x2_LOOP_END
+
+	.align 5
+
+.LZGEMM_L1x2_LOOP:
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_2
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x2_LOOP
+
+.LZGEMM_L1x2_LOOP_END:
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_2
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_1
+	KERNEL1x2_E2
+
+	b		.LZGEMM_L1x2_SUB1
+
+.LZGEMM_L1x2_SUB4:
+
+	KERNEL1x2_SUBI1
+	KERNEL1x2_SUB1
+	KERNEL1x2_SUB1
+	KERNEL1x2_SUB1
+
+	KERNEL1x2_SUB1
+	KERNEL1x2_SUB1
+	KERNEL1x2_SUB1
+	KERNEL1x2_SUB1
+
+	b		.LZGEMM_L1x2_SUB1
+
+.LZGEMM_L1x2_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL1x2_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L1x2_SAVE
+	b		.LZGEMM_L1x2_SUB2
+
+.LZGEMM_L1x2_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L1x2_SAVE
+
+.LZGEMM_L1x2_SUB2:
+
+	KERNEL1x2_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x2_SUB2
+
+.LZGEMM_L1x2_SAVE:
+
+	SAVE1x2
+
+.LZGEMM_L1x2_END:
+
+.LZGEMM_L1x1_BEGIN:
+
+
+	andi.		T1,	M,	1
+	ble		.LZGEMM_L1x1_END
+	mr		BO,	B
+	srawi.		L,	K,	3
+	ble		.LZGEMM_L1x1_SUB0
+	cmpwi		cr0,	L,	1
+	ble		.LZGEMM_L1x1_SUB4
+
+.LZGEMM_L1x1_LOOP_START:
+
+	LOAD1x1_1
+	KERNEL1x1_I1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_2
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_2
+
+	addic.		L,	L,	-2
+	ble		.LZGEMM_L1x1_LOOP_END
+
+	.align 5
+
+.LZGEMM_L1x1_LOOP:
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_2
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_2
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x1_LOOP
+
+.LZGEMM_L1x1_LOOP_END:
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_2
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_1
+	KERNEL1x1_E2
+
+	b		.LZGEMM_L1x1_SUB1
+
+.LZGEMM_L1x1_SUB4:
+
+	KERNEL1x1_SUBI1
+	KERNEL1x1_SUB1
+	KERNEL1x1_SUB1
+	KERNEL1x1_SUB1
+
+	KERNEL1x1_SUB1
+	KERNEL1x1_SUB1
+	KERNEL1x1_SUB1
+	KERNEL1x1_SUB1
+
+	b		.LZGEMM_L1x1_SUB1
+
+.LZGEMM_L1x1_SUB0:
+
+	andi.		L,	K,	7
+
+	KERNEL1x1_SUBI1
+
+	addic.		L,	L,	-1
+	ble		.LZGEMM_L1x1_SAVE
+	b		.LZGEMM_L1x1_SUB2
+
+.LZGEMM_L1x1_SUB1:
+
+	andi.		L,	K,	7
+	ble		.LZGEMM_L1x1_SAVE
+
+.LZGEMM_L1x1_SUB2:
+
+	KERNEL1x1_SUB1
+
+	addic.		L,	L,	-1
+	bgt		.LZGEMM_L1x1_SUB2
+
+.LZGEMM_L1x1_SAVE:
+
+	SAVE1x1
+
+.LZGEMM_L1x1_END:
+
+.LZGEMM_L1_END:

kernel/power/zgemv_n.S

@@ -170,6 +170,11 @@
 #define PREFETCHSIZE_C  24
 #endif
 
+#ifdef POWER8
+#define PREFETCHSIZE_A  24
+#define PREFETCHSIZE_C  24
+#endif
+
 #ifndef XCONJ
 #define FMADDR FMADD
 #define FMSUBR FNMSUB

kernel/power/zgemv_t.S

@@ -144,6 +144,12 @@
 #define PREFETCHSIZE_C   8
 #endif
 
+#ifdef POWER8
+#define PREFETCHSIZE_A  24
+#define PREFETCHSIZE_C   8
+#endif
+
+
 #if !(defined(CONJ) && defined(XCONJ))
 #define FMADDR FMADD
 #define FMSUBR FNMSUB

kernel/power/ztrmm_kernel_8x2_power8.S

@@ -0,0 +1,377 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 320
+#define ALPHA_R_SP 296(SP)
+#define ALPHA_I_SP 304(SP)
+#define FZERO	312(SP)
+#else
+#define STACKSIZE 256
+#define ALPHA_R_SP 224(SP)
+#define ALPHA_I_SP 232(SP)
+#define FZERO	240(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r10
+#define	B	r6
+#define	C	r7
+#define	LDC	r8
+#define OFFSET	r9
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#define o0	0
+#define alpha_r vs30
+#define alpha_i vs31
+
+#define KKK	r13
+#define K1	r14
+#define L	r15
+#define ALPHA	r16
+#define o24	r17
+#define T2	r19
+#define KK	r20
+#define	o8	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define o16	r27
+#define	o32	r28
+#define o48	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+	std	r13,  288(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+	stw	r13,  216(SP)
+#endif
+
+	stfd	f1,  ALPHA_R_SP
+	stfd	f2,  ALPHA_I_SP
+	stw	r0,  FZERO
+
+#ifdef linux
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)
+	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)
+	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)
+#else
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+
+#ifdef TRMMKERNEL
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+	neg	KK, OFFSET
+#endif
+#endif
+
+#include "zgemm_macros_8x2_power8.S"
+
+	cmpwi	cr0, M, 0
+	ble	.L999
+	cmpwi	cr0, N, 0
+	ble	.L999
+	cmpwi	cr0, K, 0
+	ble	.L999
+
+	slwi	LDC, LDC, ZBASE_SHIFT
+	li	PRE, 256 
+	li	o8  , 8
+	li	o16 , 16
+	li	o24 , 24
+	li	o32 , 32
+	li	o48 , 48
+
+#ifdef __64BIT__
+	addi	ALPHA, SP, 296
+#else
+	addi	ALPHA, SP, 224
+#endif
+
+	lxsdx	alpha_r, 0, ALPHA
+	lxsdx	alpha_i, o8, ALPHA
+
+	.align 4
+
+#include "ztrmm_logic_8x2_power8.S"
+
+.L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+	ld	r13,  288(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+	lwz	r13,  216(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/x86_64/KERNEL.BULLDOZER

@@ -69,24 +69,24 @@ ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_barcelona.S
 ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_barcelona.S
 
-STRSMKERNEL_LN  =  ../generic/trsm_kernel_LN.c
-STRSMKERNEL_LT  =  ../generic/trsm_kernel_LT.c
-STRSMKERNEL_RN  =  ../generic/trsm_kernel_RN.c
-STRSMKERNEL_RT  =  ../generic/trsm_kernel_RT.c
+STRSMKERNEL_LN  = strsm_kernel_LN_bulldozer.c 
+STRSMKERNEL_LT  = strsm_kernel_LT_bulldozer.c 
+STRSMKERNEL_RN  = strsm_kernel_RN_bulldozer.c 
+STRSMKERNEL_RT  = strsm_kernel_RT_bulldozer.c
 
-DTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
+DTRSMKERNEL_LN  = dtrsm_kernel_LN_bulldozer.c
 DTRSMKERNEL_LT  = dtrsm_kernel_LT_8x2_bulldozer.S
 DTRSMKERNEL_RN  = dtrsm_kernel_RN_8x2_bulldozer.S
-DTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+DTRSMKERNEL_RT  = dtrsm_kernel_RT_bulldozer.c
 
-CTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
-CTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
-CTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
-CTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+CTRSMKERNEL_LN  = ctrsm_kernel_LN_bulldozer.c
+CTRSMKERNEL_LT  = ctrsm_kernel_LT_bulldozer.c
+CTRSMKERNEL_RN  = ctrsm_kernel_RN_bulldozer.c
+CTRSMKERNEL_RT  = ctrsm_kernel_RT_bulldozer.c
 
-ZTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
-ZTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
-ZTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
-ZTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+ZTRSMKERNEL_LN  = ztrsm_kernel_LN_bulldozer.c
+ZTRSMKERNEL_LT  = ztrsm_kernel_LT_bulldozer.c
+ZTRSMKERNEL_RN  = ztrsm_kernel_RN_bulldozer.c
+ZTRSMKERNEL_RT  = ztrsm_kernel_RT_bulldozer.c
 
 

kernel/x86_64/KERNEL.HASWELL

@@ -80,7 +80,7 @@ STRSMKERNEL_RT	=  ../generic/trsm_kernel_RT.c
 
 DTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c
 DTRSMKERNEL_LT	= ../generic/trsm_kernel_LT.c
-DTRSMKERNEL_RN	= ../generic/trsm_kernel_RN.c
+DTRSMKERNEL_RN	= dtrsm_kernel_RN_haswell.c
 DTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
 
 CTRSMKERNEL_LN	= ../generic/trsm_kernel_LN.c

kernel/x86_64/KERNEL.PILEDRIVER

@@ -66,25 +66,23 @@ ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 CGEMM3MKERNEL    =  zgemm3m_kernel_8x4_barcelona.S
 ZGEMM3MKERNEL    =  zgemm3m_kernel_4x4_barcelona.S
 
-STRSMKERNEL_LN  =  ../generic/trsm_kernel_LN.c
-STRSMKERNEL_LT  =  ../generic/trsm_kernel_LT.c
-STRSMKERNEL_RN  =  ../generic/trsm_kernel_RN.c
-STRSMKERNEL_RT  =  ../generic/trsm_kernel_RT.c
+STRSMKERNEL_LN  = strsm_kernel_LN_bulldozer.c 
+STRSMKERNEL_LT  = strsm_kernel_LT_bulldozer.c 
+STRSMKERNEL_RN  = strsm_kernel_RN_bulldozer.c 
+STRSMKERNEL_RT  = strsm_kernel_RT_bulldozer.c
 
-
-DTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
+DTRSMKERNEL_LN  = dtrsm_kernel_LN_bulldozer.c
 DTRSMKERNEL_LT  = dtrsm_kernel_LT_8x2_bulldozer.S
 DTRSMKERNEL_RN  = dtrsm_kernel_RN_8x2_bulldozer.S
-DTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+DTRSMKERNEL_RT  = dtrsm_kernel_RT_bulldozer.c
 
-CTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
-CTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
-CTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
-CTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
-
-ZTRSMKERNEL_LN  = ../generic/trsm_kernel_LN.c
-ZTRSMKERNEL_LT  = ../generic/trsm_kernel_LT.c
-ZTRSMKERNEL_RN  = ../generic/trsm_kernel_RN.c
-ZTRSMKERNEL_RT  = ../generic/trsm_kernel_RT.c
+CTRSMKERNEL_LN  = ctrsm_kernel_LN_bulldozer.c
+CTRSMKERNEL_LT  = ctrsm_kernel_LT_bulldozer.c
+CTRSMKERNEL_RN  = ctrsm_kernel_RN_bulldozer.c
+CTRSMKERNEL_RT  = ctrsm_kernel_RT_bulldozer.c
 
+ZTRSMKERNEL_LN  = ztrsm_kernel_LN_bulldozer.c
+ZTRSMKERNEL_LT  = ztrsm_kernel_LT_bulldozer.c
+ZTRSMKERNEL_RN  = ztrsm_kernel_RN_bulldozer.c
+ZTRSMKERNEL_RT  = ztrsm_kernel_RT_bulldozer.c