Restore -march flag for Android builds

fixes #2419 - renewed discussion in #2112 suggests removal of the option was primarily aimed at non-Android builds
Merge pull request #2417 from marxin/make-ctest-verbose-for-drone
2020-02-16 17:32:13 +01:00 · 2020-02-15 21:57:41 +01:00 · 2020-02-15 21:57:03 +01:00 · 2020-02-15 21:56:16 +01:00 · 2020-02-15 19:29:14 +01:00 · 2020-02-14 23:07:43 +01:00
1463 changed files with 118403 additions and 24006 deletions
@@ -0,0 +1,143 @@
+---
+kind: pipeline
+name: arm64_gcc_make
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:19.04
+  environment:
+    CC: gcc
+    COMMON_FLAGS: 'DYNAMIC_ARCH=1 TARGET=ARMV8 NUM_THREADS=32'
+  commands:
+    - echo "MAKE_FLAGS:= $COMMON_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC gfortran perl
+    - $CC --version
+    - make QUIET_MAKE=1 $COMMON_FLAGS
+    - make -C test $COMMON_FLAGS
+    - make -C ctest $COMMON_FLAGS
+    - make -C utest $COMMON_FLAGS
+
+---
+kind: pipeline
+name: arm32_gcc_make
+
+platform:
+  os: linux
+  arch: arm
+
+steps:
+- name: Build and Test
+  image: ubuntu:19.04
+  environment:
+    CC: gcc
+    COMMON_FLAGS: 'DYNAMIC_ARCH=1 TARGET=ARMV6 NUM_THREADS=32'
+  commands:
+    - echo "MAKE_FLAGS:= $COMMON_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC gfortran perl
+    - $CC --version
+    - make QUIET_MAKE=1 $COMMON_FLAGS
+    - make -C test $COMMON_FLAGS
+    - make -C ctest $COMMON_FLAGS
+    - make -C utest $COMMON_FLAGS
+
+---
+kind: pipeline
+name: arm64_clang_make
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    COMMON_FLAGS: 'DYNAMIC_ARCH=1 TARGET=ARMV8 NUM_THREADS=32'
+  commands:
+    - echo "MAKE_FLAGS:= $COMMON_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC gfortran perl
+    - $CC --version
+    - make QUIET_MAKE=1 $COMMON_FLAGS
+    - make -C test $COMMON_FLAGS
+    - make -C ctest $COMMON_FLAGS
+    - make -C utest $COMMON_FLAGS
+
+---
+kind: pipeline
+name: arm32_clang_cmake
+
+platform:
+  os: linux
+  arch: arm
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CMAKE_FLAGS: '-DDYNAMIC_ARCH=1 -DTARGET=ARMV6 -DNUM_THREADS=32 -DNOFORTRAN=ON -DBUILD_WITHOUT_LAPACK=ON'
+  commands:
+    - echo "CMAKE_FLAGS:= $CMAKE_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC g++ perl cmake
+    - $CC --version
+    - mkdir build && cd build
+    - cmake $CMAKE_FLAGS ..
+    - make -j
+    - ctest -V
+
+---
+kind: pipeline
+name: arm64_gcc_cmake
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: gcc
+    CMAKE_FLAGS: '-DDYNAMIC_ARCH=1 -DTARGET=ARMV8 -DNUM_THREADS=32 -DNOFORTRAN=ON -DBUILD_WITHOUT_LAPACK=ON'
+  commands:
+    - echo "CMAKE_FLAGS:= $CMAKE_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC g++ perl cmake
+    - $CC --version
+    - mkdir build && cd build
+    - cmake $CMAKE_FLAGS ..
+    - make -j
+    - ctest -V
+
+---
+kind: pipeline
+name: arm64_clang_cmake
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: Build and Test
+  image: ubuntu:18.04
+  environment:
+    CC: clang
+    CMAKE_FLAGS: '-DDYNAMIC_ARCH=1 -DTARGET=ARMV8 -DNUM_THREADS=32 -DNOFORTRAN=ON -DBUILD_WITHOUT_LAPACK=ON'
+  commands:
+    - echo "CMAKE_FLAGS:= $CMAKE_FLAGS"
+    - apt-get update -y
+    - apt-get install -y make $CC g++ perl cmake
+    - $CC --version
+    - mkdir build && cd build
+    - cmake $CMAKE_FLAGS ..
+    - make -j
+    - ctest -V
@@ -87,4 +87,5 @@ build.*
 *.swp
 benchmark/*.goto
 benchmark/smallscaling
-
+CMakeCache.txt
+CMakeFiles/*
@@ -17,7 +17,7 @@ matrix:
        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32"
      script:
        - set -e
-        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
+        - make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
        - make -C test $COMMON_FLAGS $BTYPE
        - make -C ctest $COMMON_FLAGS $BTYPE
        - make -C utest $COMMON_FLAGS $BTYPE
@@ -25,6 +25,15 @@ matrix:
        - TARGET_BOX=LINUX64
        - BTYPE="BINARY=64"

+    - <<: *test-ubuntu
+      os: linux-ppc64le
+      before_script:
+        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=POWER8 NUM_THREADS=32"
+      env:
+        # for matrix annotation only
+        - TARGET_BOX=PPC64LE_LINUX
+        - BTYPE="BINARY=64 USE_OPENMP=1"
+
    - <<: *test-ubuntu
      env:
        - TARGET_BOX=LINUX64
@@ -117,7 +126,7 @@ matrix:
    - <<: *test-alpine
      env:
        - TARGET_BOX=LINUX64_MUSL
-        - BTYPE="BINARY=64 NO_AFFINITY=1 USE_OPENMP=0 NO_LAPACK=0 TARGET=core2"
+        - BTYPE="BINARY=64 NO_AFFINITY=1 USE_OPENMP=0 NO_LAPACK=0 TARGET=CORE2"

    - &test-cmake
      os: linux
@@ -149,55 +158,27 @@ matrix:

    - &test-macos
      os: osx
-      osx_image: xcode8
+      osx_image: xcode10.1
      before_script:
-        - COMMON_FLAGS="DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32"
+        - COMMON_FLAGS="DYNAMIC_ARCH=1 NUM_THREADS=32"
        - brew update
-        - brew install gcc # for gfortran
+        - brew install gcc@8 # for gfortran
      script:
        - travis_wait 45 make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE
      env:
-        - BTYPE="BINARY=64 INTERFACE64=1"
+        - BTYPE="TARGET=NEHALEM BINARY=64 INTERFACE64=1 FC=gfortran-8"

    - <<: *test-macos
+      osx_image: xcode10.0
      env:
-        - BTYPE="BINARY=32"
+        - BTYPE="TARGET=NEHALEM BINARY=32 NOFORTRAN=1"

-    - &emulated-arm
-      dist: trusty
-      sudo: required
-      services: docker
-      env: IMAGE_ARCH=arm32 TARGET_ARCH=ARMV6 COMPILER=gcc
-      name: "Emulated Build for ARMV6 with gcc"
-      before_install: sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset
-      script: |
-        echo "FROM openblas/alpine:${IMAGE_ARCH}
-        COPY . /tmp/openblas
-        RUN mkdir /tmp/openblas/build                             &&  \
-            cd /tmp/openblas/build                                &&  \
-            CC=${COMPILER} cmake -D DYNAMIC_ARCH=OFF                  \
-                                 -D TARGET=${TARGET_ARCH}             \
-                                 -D BUILD_SHARED_LIBS=ON              \
-                                 -D BUILD_WITHOUT_LAPACK=ON           \
-                                 -D BUILD_WITHOUT_CBLAS=ON            \
-                                 -D CMAKE_BUILD_TYPE=Release ../  &&  \
-            cmake --build ." > Dockerfile
-        docker build .
-    - <<: *emulated-arm
-      env: IMAGE_ARCH=arm32 TARGET_ARCH=ARMV6 COMPILER=clang
-      name: "Emulated Build for ARMV6 with clang"
-    - <<: *emulated-arm
-      env: IMAGE_ARCH=arm64 TARGET_ARCH=ARMV8 COMPILER=gcc
-      name: "Emulated Build for ARMV8 with gcc"
-    - <<: *emulated-arm
-      env: IMAGE_ARCH=arm64 TARGET_ARCH=ARMV8 COMPILER=clang
-      name: "Emulated Build for ARMV8 with clang"
-
-  allow_failures:
-    - env: IMAGE_ARCH=arm32 TARGET_ARCH=ARMV6 COMPILER=gcc
-    - env: IMAGE_ARCH=arm32 TARGET_ARCH=ARMV6 COMPILER=clang
-    - env: IMAGE_ARCH=arm64 TARGET_ARCH=ARMV8 COMPILER=gcc
-    - env: IMAGE_ARCH=arm64 TARGET_ARCH=ARMV8 COMPILER=clang
+    - <<: *test-macos
+      osx_image: xcode10.1
+      env:
+        - CC="/Applications/Xcode-10.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -isysroot /Applications/Xcode-10.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS12.1.sdk"
+        - CFLAGS="-O2 -Wno-macro-redefined -isysroot /Applications/Xcode-10.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS12.1.sdk -arch arm64 -miphoneos-version-min=10.0"
+        - BTYPE="TARGET=ARMV8 BINARY=64 HOSTCC=clang NOFORTRAN=1"

 # whitelist
 branches:
@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.5)
 project(OpenBLAS C ASM)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 3)
-set(OpenBLAS_PATCH_VERSION 4)
+set(OpenBLAS_PATCH_VERSION 9.dev)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")

 # Adhere to GNU filesystem layout conventions
@@ -20,9 +20,14 @@ if(MSVC)
 option(BUILD_WITHOUT_LAPACK "Do not build LAPACK and LAPACKE (Only BLAS or CBLAS)" ON)
 endif()
 option(BUILD_WITHOUT_CBLAS "Do not build the C interface (CBLAS) to the BLAS functions" OFF)
-option(DYNAMIC_ARCH "Include support for multiple CPU targets, with automatic selection at runtime (x86/x86_64 only)" OFF)
-option(DYNAMIC_OLDER "Include specific support for older cpu models (Penryn,Dunnington,Atom,Nano,Opteron) with DYNAMIC_ARCH" OFF)
+option(DYNAMIC_ARCH "Include support for multiple CPU targets, with automatic selection at runtime (x86/x86_64, aarch64 or ppc only)" OFF)
+option(DYNAMIC_OLDER "Include specific support for older x86 cpu models (Penryn,Dunnington,Atom,Nano,Opteron) with DYNAMIC_ARCH" OFF)
 option(BUILD_RELAPACK "Build with ReLAPACK (recursive implementation of several LAPACK functions on top of standard LAPACK)" OFF)
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+option(NO_AFFINITY "Disable support for CPU affinity masks to avoid binding processes from e.g. R or numpy/scipy to a single core" ON)
+else()
+set(NO_AFFINITY 1)
+endif()

 # Add a prefix or suffix to all exported symbol names in the shared library.
 # Avoids conflicts with other BLAS libraries, especially when using
@@ -42,6 +47,19 @@ endif()

 #######

+if(MSVC AND MSVC_STATIC_CRT)
+    set(CompilerFlags
+            CMAKE_CXX_FLAGS
+            CMAKE_CXX_FLAGS_DEBUG
+            CMAKE_CXX_FLAGS_RELEASE
+            CMAKE_C_FLAGS
+            CMAKE_C_FLAGS_DEBUG
+            CMAKE_C_FLAGS_RELEASE
+            )
+    foreach(CompilerFlag ${CompilerFlags})
+      string(REPLACE "/MD" "/MT" ${CompilerFlag} "${${CompilerFlag}}")
+    endforeach()
+endif()

 message(WARNING "CMake support is experimental. It does not yet support all build options and may not produce the same Makefiles that OpenBLAS ships with.")

@@ -62,10 +80,10 @@ endif ()

 set(SUBDIRS	${BLASDIRS})
 if (NOT NO_LAPACK)
-  list(APPEND SUBDIRS lapack)
  if(BUILD_RELAPACK)
    list(APPEND SUBDIRS relapack/src)
  endif()
+  list(APPEND SUBDIRS lapack)
 endif ()

 # set which float types we want to build for
@@ -134,7 +152,7 @@ endif ()

 # Only generate .def for dll on MSVC and always produce pdb files for debug and release
 if(MSVC)
-  if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} LESS 3.4)
+  if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} VERSION_LESS 3.4)
    set(OpenBLAS_DEF_FILE "${PROJECT_BINARY_DIR}/openblas.def")
  endif()
  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /Zi")
@@ -149,15 +167,9 @@ if (${DYNAMIC_ARCH})
  endforeach()
 endif ()

-# Only build shared libs for MSVC
-if (MSVC)
-  set(BUILD_SHARED_LIBS ON)
-endif()
-
-
 # add objects to the openblas lib
 add_library(${OpenBLAS_LIBNAME} ${LA_SOURCES} ${LAPACKE_SOURCES} ${RELA_SOURCES} ${TARGET_OBJS} ${OpenBLAS_DEF_FILE})
-target_include_directories(${OpenBLAS_LIBNAME} INTERFACE $<INSTALL_INTERFACE:include>)
+target_include_directories(${OpenBLAS_LIBNAME} INTERFACE $<INSTALL_INTERFACE:include/openblas${SUFFIX64}>)

 # Android needs to explicitly link against libm
 if(ANDROID)
@@ -166,7 +178,7 @@ endif()

 # Handle MSVC exports
 if(MSVC AND BUILD_SHARED_LIBS)
-  if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} LESS 3.4)
+  if (${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} VERSION_LESS 3.4)
    include("${PROJECT_SOURCE_DIR}/cmake/export.cmake")
  else()
    # Creates verbose .def file (51KB vs 18KB)
@@ -199,7 +211,8 @@ if (USE_THREAD)
  target_link_libraries(${OpenBLAS_LIBNAME} ${CMAKE_THREAD_LIBS_INIT})
 endif()

-if (MSVC OR NOT NOFORTRAN)
+#if (MSVC OR NOT NOFORTRAN)
+if (NOT NO_CBLAS)
  # Broken without fortran on unix
  add_subdirectory(utest)
 endif()
@@ -217,6 +230,14 @@ set_target_properties(${OpenBLAS_LIBNAME} PROPERTIES
  SOVERSION ${OpenBLAS_MAJOR_VERSION}
 )

+if (BUILD_SHARED_LIBS AND BUILD_RELAPACK)
+  if (NOT MSVC)
+    target_link_libraries(${OpenBLAS_LIBNAME} "-Wl,-allow-multiple-definition")
+  else()
+    target_link_libraries(${OpenBLAS_LIBNAME} "/FORCE:MULTIPLE")
+  endif()
+endif()
+
 if (BUILD_SHARED_LIBS AND NOT ${SYMBOLPREFIX}${SYMBOLSUFIX} STREQUAL "")
 if (NOT DEFINED ARCH)
  set(ARCH_IN "x86_64")
@@ -314,7 +335,7 @@ install (FILES ${OPENBLAS_CONFIG_H} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 if(NOT NOFORTRAN)
  message(STATUS "Generating f77blas.h in ${CMAKE_INSTALL_INCLUDEDIR}")

-  set(F77BLAS_H ${CMAKE_BINARY_DIR}/f77blas.h)
+  set(F77BLAS_H ${CMAKE_BINARY_DIR}/generated/f77blas.h)
  file(WRITE  ${F77BLAS_H} "#ifndef OPENBLAS_F77BLAS_H\n")
  file(APPEND ${F77BLAS_H} "#define OPENBLAS_F77BLAS_H\n")
  file(APPEND ${F77BLAS_H} "#include \"openblas_config.h\"\n")
@@ -327,10 +348,11 @@ endif()
 if(NOT NO_CBLAS)
 	message (STATUS "Generating cblas.h in ${CMAKE_INSTALL_INCLUDEDIR}")

+	set(CBLAS_H ${CMAKE_BINARY_DIR}/generated/cblas.h)
 	file(READ ${CMAKE_CURRENT_SOURCE_DIR}/cblas.h CBLAS_H_CONTENTS)
 	string(REPLACE "common" "openblas_config" CBLAS_H_CONTENTS_NEW "${CBLAS_H_CONTENTS}")
-	file(WRITE ${CMAKE_BINARY_DIR}/cblas.tmp "${CBLAS_H_CONTENTS_NEW}")
-	install (FILES ${CMAKE_BINARY_DIR}/cblas.tmp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} RENAME cblas.h)
+	file(WRITE ${CBLAS_H} "${CBLAS_H_CONTENTS_NEW}")
+	install (FILES ${CBLAS_H} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 endif()

 if(NOT NO_LAPACKE)
@@ -167,4 +167,16 @@ In chronological order:
  * [2017-02-26] ztrmm kernel for IBM z13
  * [2017-03-13] strmm and ctrmm kernel for IBM z13
  * [2017-09-01] initial Blas Level-1,2 (double precision) for IBM z13
+  * [2018-03-07] added missing Blas Level 1-2  (double precision) simd codes
+  * [2019-02-01] added missing Blas Level-1,2 (single precision)  simd codes
+  * [2019-03-14] power9 dgemm/dtrmm kernel
+  * [2019-04-29] power9 sgemm/strmm kernel 

+* Jiachen Wang <https://github.com/wjc404>
+  * [2019-07-29] optimize AVX2 DGEMM
+  * [2019-10-20] AVX512 DGEMM kernel (4x8)
+  * [2019-11-06] optimize AVX512 SGEMM
+  * [2019-11-12] AVX512 CGEMM & ZGEMM kernels
+  * [2019-12-23] optimize AVX2 CGEMM and ZGEMM
+  * [2019-12-30] AVX2 CGEMM3M & ZGEMM3M kernels
+  * [2020-01-07] optimize AVX2 SGEMM and STRMM
@@ -1,4 +1,211 @@
 OpenBLAS ChangeLog
+====================================================================
+Version 0.3.8
+ 9-Feb-2020
+
+common: 
+`	* LAPACK has been updated to 3.9.0 (plus patches up to 
+	  January 2nd, 2020)
+	* CMAKE support has been improved in several areas including
+	  cross-compilation
+	* a thread race condition in the GEMM3M kernels was resolved
+	* the "generic" (plain C) gemm beta kernel used by many targets
+	  has been sped up
+	* an optimized version of the LAPACK trtrs functions has been added
+	* an incompatibilty between the LAPACK tests and the OpenBLAS
+	  implementation of XERBLA was resolved, removing the numerous
+	  warnings about wrong error exits in the former 
+	* support for NetBSD has been added
+	* support for compilation with g95 and non-GNU versions of ld
+	  has been improved
+	* support for compilation with (upcoming) gcc 10 has been added
+
+POWER:
+	* worked around miscompilation of several POWER8 and POWER9 
+	  kernels by older versions of gcc
+	* added support for big-endian POWER8 and for compilation on AIX
+	* corrected bugs in the big-endian support for PPC440 and PPC970
+	* DYNAMIC_ARCH support is now available in CMAKE builds as well
+
+ARMV8:
+	* performance of DGEMM_BETA and SGEMM_NCOPY has been improved
+	* compilation for 32bit works again 
+	* performance of the RPCC function has been improved
+	* improved performance on small systems
+	* DYNAMIC_ARCH support is now available in CMAKE builds as well
+	* cross-compilation from OSX to IOS was simplified
+ 
+x86_64:
+	* a new AVX512 DGEMM kernel was added and the AVX512 SGEMM kernel
+	  was significantly improved
+	* optimized AVX512 kernels for CGEMM and ZGEMM have been added
+	* AVX2 kernels for STRMM, SGEMM, and CGEMM have been significantly
+	  sped up and optimized CGEMM3M and ZGEMM3M kernels have been added 
+	* added support for QEMU virtual cpus
+	* a compilation problem with PGI and SUN compilers was fixed
+	* Intel "Goldmont plus" is now autodetected
+	* a potential crash on program exit on MS Windows has been fixed
+
+x86:
+	* an unwanted case sensitivity in the implementation of LSAME
+  	  on older 32bit AMD cpus was fixed
+
+zarch:
+	* Z15 is now supported as Z14
+	* DYNAMIC_ARCH is now available on ZARCH as well
+
+====================================================================
+Version 0.3.7
+11-Aug 2019
+
+common:
+  * having the gmake special variables TARGET_ARCH or TARGET_MACH
+    defined no longer causes build failures in ctest or utest
+  * defining NO_AFFINITY or USE_TLS to 0 in gmake builds no longer
+    has the same effect as setting them to 1
+  * a new test program was added to allow checking the library for
+    thread safety
+  * a new option USE_LOCKING was added to ensure thread safety when
+    OpenBLAS itself is built without multithreading but will be 
+    called from multiple threads.
+  * a build failure on Linux with glibc versions earlier than 2.5
+    was fixed
+  * a runtime error with CPU enumeration (and NO_AFFINITY not set)
+    on glibc 2.6 was fixed
+  * NO_AFFINITY was added to the CMAKE options (and defaults to being
+    active on Linux, as in the gmake builds)
+
+x86_64:
+  * the build-time logic for detection of AVX512 availability in
+    the processor and compiler was fixed
+  * gmake builds on OSX now set the internal name of the library to 
+    libopenblas.0.dylib (consistent with CMAKE)
+  * the Haswell DGEMM kernel received a significant speedup through
+    improved prefetch and load instructions
+  * performance of DGEMM, DTRMM, DTRSM and ZDOT on Zen/Zen2 was markedly
+    increased by avoiding vpermpd instructions
+  * the SKYLAKEX (AVX512) DGEMM helper functions have now been disabled
+    to fix remaining errors in DGEMM, DSYMM and DTRMM
+
+POWER:
+  * added support for building on FreeBSD/powerpc64 and FreeBSD/ppc970
+  * added optimized kernels for POWER9 SGEMM and STRMM
+
+ARMV7:
+  * fixed the softfp implementations of xAMAX and IxAMAX
+  * removed the predefined -march= flags on both ARMV5 and ARMV6 as
+    they were appropriate for only a subset of platforms
+
+====================================================================
+Version 0.3.6
+29-Apr-2019
+
+common:
+	* the build tools now check that a given cpu TARGET is actually valid
+	* the build-time check of system features (c_check) has been made
+  	  less dependent on particular perl features (this should mainly
+  	  benefit building on Windows)
+	* several problem with the ReLAPACK integration were fixed,
+	  including INTERFACE64 support and building a shared library
+	* building with CMAKE on BSD systems was improved
+	* a non-absolute SUM function was added based on the
+  	  existing optimized code for ASUM
+	* CBLAS interfaces to the IxMIN and IxMAX functions were added
+	* a name clash between LAPACKE and BOOST headers was resolved
+	* CMAKE builds with OpenMP failed to include the appropriate getrf_parallel
+	  kernels
+	* a crash on thread (key) deletion with the USE_TLS=1 memory management
+	  option was fixed
+	* restored several earlier fixes, in particular for OpenMP performance,
+  	  building on BSD, and calling fork on CYGWIN, which had inadvertently
+  	  been dropped in the 0.3.3 rewrite of the memory management code.
+
+x86_64:
+	* the AVX512 DGEMM kernel has been disabled again due to unsolved problems
+	* building with old versions of MSVC was fixed
+	* it is now possible to build a static library on Windows with CMAKE
+	* accessing environment variables on CYGWIN at run time was fixed
+	* the CMAKE build system now recognizes 32bit userspace on 64bit hardware
+	* Intel "Denverton" atom and Hygon "Dhyana" zen CPUs are now autodetected
+	* building for DYNAMIC_ARCH with a DYNAMIC_LIST of targets is now supported
+  	  with CMAKE as well
+	* building for DYNAMIC_ARCH with GENERIC as the default target is now supported
+	* a buffer overflow in the SSE GEMM kernel for Intel Nano targets was fixed
+	* assembly bugs involving undeclared modification of input operands were fixed
+  	  in the AXPY, DOT, GEMV, GER, SCAL, SYMV and TRSM microkernels for Nehalem,
+	  Sandybridge, Haswell, Bulldozer and Piledriver. These would typically cause
+	  test failures or segfaults when compiled with recent versions of gcc from 8 onward.
+	* a similar bug was fixed in the blas_quickdivide code used to split workloads
+	  in most functions
+	* a bug in the IxMIN implementation for the GENERIC target made it return the result of IxMAX
+	* fixed building on SkylakeX systems when either the compiler or the (emulated) operating
+	  environment does not support AVX512
+	* improved GEMM performance on ZEN targets
+
+x86:
+	* build failures caused by the recently added checks for AVX512 were fixed
+	* an inline assembly bug involving undeclared modification of an input argument was
+  	  fixed in the blas_quickdivide code used to split workloads in most functions
+	* a bug in the IMIN implementation for the GENERIC target made it return the result of IMAX
+
+MIPS32:
+	* a bug in the IMIN implementation made it return the result of IMAX
+
+POWER:
+	* single precision BLAS1/2 functions have received optimized POWER8 kernels
+	* POWER9 is now a separate target, with an optimized DGEMM/DTRMM kernel
+	* building on PPC970 systems under OSX Leopard or Tiger is now supported
+	* out-of-bounds memory accesses in the gemm_beta microkernels were fixed
+	* building a shared library on AIX is now supported for POWER6
+	* DYNAMIC_ARCH support has been added for POWER6 and newer
+
+ARMv7:
+	* corrected xDOT behaviour with zero INC_X or INC_Y
+	* a bug in the IMIN implementation made it return the result of IMAX
+
+ARMv8:
+	* added support for HiSilicon TSV110 cpus
+	* the CMAKE build system now recognizes 32bit userspace on 64bit hardware
+	* cross-compilation with CMAKE now works again
+	* a bug in the IMIN implementation made it return the result of IMAX
+	* ARMV8 builds with the BINARY=32 option are now automatically handled as ARMV7
+
+IBM Z:
+	* optimized microkernels for single precicion BLAS1/2 functions have been added
+	  for both Z13 and Z14
+
+====================================================================
+Version 0.3.5
+31-Dec-2018
+
+common:
+	* loop unrolling in TRMV has been enabled again.
+	* A domain error in the thread workload distribution for SYRK
+	  has been fixed.
+	* gmake builds will now automatically add -fPIC to the build
+	  options if the platform requires it.
+	* a pthreads key leakage (and associate crash on dlclose) in
+	  the USE_TLS codepath was fixed.
+	* building of the utest cases on systems that do not provide
+	  an implementation of complex.h was fixed.
+	  
+x86_64:
+	* the SkylakeX code was changed to compile on OSX.
+	* unwanted application of the -march=skylake-avx512 option
+	  to the common code parts of a DYNAMIC_ARCH build was fixed.
+	* improved performance of SGEMM for small workloads on Skylake X.
+	* performance of SGEMM and DGEMM was improved on Haswell.
+
+ARMV8:
+	* a configuration error that broke the CNRM2 kernel was corrected.
+	* compilation of the GEMM kernels with CMAKE was fixed.
+	* DYNAMIC_ARCH builds are now available with CMAKE as well.
+	* using CMAKE for cross-compilation to the new cpu TARGETs
+	  introduced in 0.3.4 now works.
+	  
+POWER:
+	* a problem in cpu autodetection for AIX has been corrected.
+	
 ====================================================================
 Version 0.3.4
 02-Dec-2018
@@ -34,7 +34,7 @@ endif

 LAPACK_NOOPT := $(filter-out -O0 -O1 -O2 -O3 -Ofast,$(LAPACK_FFLAGS))

-SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench
+SUBDIRS_ALL = $(SUBDIRS) test ctest utest exports benchmark ../laswp ../bench cpp_thread_test

 .PHONY : all libs netlib $(RELA) test ctest shared install
 .NOTPARALLEL : all libs $(RELA) prof lapack-test install blas-test
@@ -96,7 +96,7 @@ endif
 	@echo

 shared :
-ifndef NO_SHARED
+ifneq ($(NO_SHARED), 1)
 ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS Android Haiku))
 	@$(MAKE) -C exports so
 	@ln -fs $(LIBSONAME) $(LIBPREFIX).so
@@ -109,6 +109,7 @@ endif
 ifeq ($(OSNAME), Darwin)
 	@$(MAKE) -C exports dyn
 	@ln -fs $(LIBDYNNAME) $(LIBPREFIX).dylib
+	@ln -fs $(LIBDYNNAME) $(LIBPREFIX).$(MAJOR_VERSION).dylib
 endif
 ifeq ($(OSNAME), WINNT)
 	@$(MAKE) -C exports dll
@@ -123,15 +124,18 @@ ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN)))
 	touch $(LIBNAME)
 ifndef NO_FBLAS
 	$(MAKE) -C test all
-	$(MAKE) -C utest all
 endif
+	$(MAKE) -C utest all
 ifndef NO_CBLAS
 	$(MAKE) -C ctest all
+ifeq ($(CPP_THREAD_SAFETY_TEST), 1)
+	$(MAKE) -C cpp_thread_test all
+endif
 endif
 endif

 libs :
-ifeq ($(CORE), UNKOWN)
+ifeq ($(CORE), UNKNOWN)
 	$(error OpenBLAS: Detecting CPU failed. Please set TARGET explicitly, e.g. make TARGET=your_cpu_target. Please read README for the detail.)
 endif
 ifeq ($(NOFORTRAN), 1)
@@ -243,21 +247,21 @@ prof_lapack : lapack_prebuild

 lapack_prebuild :
 ifeq ($(NOFORTRAN), $(filter 0,$(NOFORTRAN)))
-	-@echo "FORTRAN     = $(FC)" > $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "OPTS        = $(LAPACK_FFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "FC          = $(FC)" > $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "FFLAGS      = $(LAPACK_FFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "POPTS       = $(LAPACK_FPFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "NOOPT       = -O0 $(LAPACK_NOOPT)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "FFLAGS_NOOPT       = -O0 $(LAPACK_NOOPT)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "PNOOPT      = $(LAPACK_FPFLAGS) -O0" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "LOADOPTS    = $(FFLAGS) $(EXTRALIB)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "LDFLAGS     = $(FFLAGS) $(EXTRALIB)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "CC          = $(CC)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "override CFLAGS      = $(LAPACK_CFLAGS)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "override ARCH        = $(AR)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "ARCHFLAGS   = $(ARFLAGS) -ru" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "AR          = $(AR)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "ARFLAGS     = $(ARFLAGS) -ru" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "RANLIB      = $(RANLIB)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "LAPACKLIB   = ../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "TMGLIB      = ../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "LAPACKLIB   = ../../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "TMGLIB      = ../../../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "BLASLIB     = ../../../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
-	-@echo "LAPACKELIB  = ../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
+	-@echo "LAPACKELIB  = ../../../$(LIBNAME)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "LAPACKLIB_P = ../$(LIBNAME_P)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "SUFFIX      = $(SUFFIX)" >> $(NETLIB_LAPACK_DIR)/make.inc
 	-@echo "PSUFFIX     = $(PSUFFIX)" >> $(NETLIB_LAPACK_DIR)/make.inc
@@ -315,7 +319,7 @@ lapack-test :
 ifneq ($(CROSS), 1)
 	( cd $(NETLIB_LAPACK_DIR)/INSTALL; make all; ./testlsame; ./testslamch; ./testdlamch; \
        ./testsecond; ./testdsecnd; ./testieee; ./testversion )
-	(cd $(NETLIB_LAPACK_DIR); ./lapack_testing.py -r )
+	(cd $(NETLIB_LAPACK_DIR); ./lapack_testing.py -r -b TESTING)
 endif

 lapack-runtest:
@@ -1,7 +1,7 @@
 ifeq ($(CORE), $(filter $(CORE),ARMV7 CORTEXA9 CORTEXA15))
 ifeq ($(OSNAME), Android)
-CCOMMON_OPT += -mfpu=neon  -march=armv7-a
-FCOMMON_OPT += -mfpu=neon  -march=armv7-a
+CCOMMON_OPT += -mfpu=neon -march=armv7-a
+FCOMMON_OPT += -mfpu=neon -march=armv7-a
 else
 CCOMMON_OPT += -mfpu=vfpv3 -march=armv7-a
 FCOMMON_OPT += -mfpu=vfpv3 -march=armv7-a
@@ -9,11 +9,6 @@ endif
 endif

 ifeq ($(CORE), ARMV6)
-CCOMMON_OPT += -mfpu=vfp -march=armv6
-FCOMMON_OPT += -mfpu=vfp -march=armv6
-endif
-
-ifeq ($(CORE), ARMV5)
-CCOMMON_OPT += -march=armv5
-FCOMMON_OPT += -march=armv5
+CCOMMON_OPT += -mfpu=vfp
+FCOMMON_OPT += -mfpu=vfp
 endif
@@ -30,11 +30,19 @@ FCOMMON_OPT += -march=armv8-a -mtune=thunderx
 endif

 ifeq ($(CORE), FALKOR)
-CCOMMON_OPT += -march=armv8.1-a -mtune=falkor
-FCOMMON_OPT += -march=armv8.1-a -mtune=falkor
+CCOMMON_OPT += -march=armv8-a -mtune=falkor
+FCOMMON_OPT += -march=armv8-a -mtune=falkor
 endif

 ifeq ($(CORE), THUNDERX2T99)
 CCOMMON_OPT += -march=armv8.1-a -mtune=thunderx2t99
 FCOMMON_OPT += -march=armv8.1-a -mtune=thunderx2t99
 endif
+
+ifeq ($(GCCVERSIONGTEQ9), 1)
+ifeq ($(CORE), TSV110)
+CCOMMON_OPT += -march=armv8.2-a -mtune=tsv110
+FCOMMON_OPT += -march=armv8.2-a -mtune=tsv110
+endif
+endif
+
@@ -51,6 +51,7 @@ endif
 ifneq ($(OSNAME), AIX)
 ifndef NO_LAPACKE
 	@echo Copying LAPACKE header files to $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)
+	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapack.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapack.h"
 	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke.h"
 	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_config.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_config.h"
 	@-install -pm644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_mangling_with_flags.h.in "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_mangling.h"
@@ -58,14 +59,14 @@ ifndef NO_LAPACKE
 endif

 #for install static library
-ifndef NO_STATIC
+ifneq ($(NO_STATIC),1)
 	@echo Copying the static library to $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
 	@install -pm644 $(LIBNAME) "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)"
 	@cd "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)" ; \
 	ln -fs $(LIBNAME) $(LIBPREFIX).$(LIBSUFFIX)
 endif
 #for install shared library
-ifndef NO_SHARED
+ifneq ($(NO_SHARED),1)
 	@echo Copying the shared library to $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
 ifeq ($(OSNAME), $(filter $(OSNAME),Linux SunOS Android Haiku))
 	@install -pm755 $(LIBSONAME) "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)"
@@ -83,7 +84,8 @@ ifeq ($(OSNAME), Darwin)
 	@-cp $(LIBDYNNAME) "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)"
 	@-install_name_tool -id "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBDYNNAME)" "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)/$(LIBDYNNAME)"
 	@cd "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)" ; \
-	ln -fs $(LIBDYNNAME) $(LIBPREFIX).dylib
+	ln -fs $(LIBDYNNAME) $(LIBPREFIX).dylib ; \
+	ln -fs $(LIBDYNNAME) $(LIBPREFIX).$(MAJOR_VERSION).dylib
 endif
 ifeq ($(OSNAME), WINNT)
 	@-cp $(LIBDLLNAME) "$(DESTDIR)$(OPENBLAS_BINARY_DIR)"
@@ -99,6 +101,7 @@ else
 #install on AIX has different options syntax
 ifndef NO_LAPACKE
 	@echo Copying LAPACKE header files to $(DESTDIR)$(OPENBLAS_INCLUDE_DIR)
+	@-installbsd -c -m 644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapack.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapack.h"
 	@-installbsd -c -m 644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke.h"
 	@-installbsd -c -m 644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_config.h "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_config.h"
 	@-installbsd -c -m 644 $(NETLIB_LAPACK_DIR)/LAPACKE/include/lapacke_mangling_with_flags.h.in "$(DESTDIR)$(OPENBLAS_INCLUDE_DIR)/lapacke_mangling.h"
@@ -106,14 +109,14 @@ ifndef NO_LAPACKE
 endif

 #for install static library
-ifndef NO_STATIC
+ifneq ($(NO_STATIC),1)
 	@echo Copying the static library to $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
 	@installbsd -c -m 644 $(LIBNAME) "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)"
 	@cd "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)" ; \
 	ln -fs $(LIBNAME) $(LIBPREFIX).$(LIBSUFFIX)
 endif
 #for install shared library
-ifndef NO_SHARED
+ifneq ($(NO_SHARED),1)
 	@echo Copying the shared library to $(DESTDIR)$(OPENBLAS_LIBRARY_DIR)
 	@installbsd -c -m 755 $(LIBSONAME) "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)"
 	@cd "$(DESTDIR)$(OPENBLAS_LIBRARY_DIR)" ; \
@@ -138,7 +141,7 @@ endif
 	@echo "SET(OpenBLAS_VERSION \"${VERSION}\")" > "$(DESTDIR)$(OPENBLAS_CMAKE_DIR)/$(OPENBLAS_CMAKE_CONFIG)"
 	@echo "SET(OpenBLAS_INCLUDE_DIRS ${OPENBLAS_INCLUDE_DIR})" >> "$(DESTDIR)$(OPENBLAS_CMAKE_DIR)/$(OPENBLAS_CMAKE_CONFIG)"

-ifndef NO_SHARED
+ifneq ($(NO_SHARED),1)
 #ifeq logical or
 ifeq ($(OSNAME), $(filter $(OSNAME),Linux FreeBSD NetBSD OpenBSD DragonFly))
 	@echo "SET(OpenBLAS_LIBRARIES ${OPENBLAS_LIBRARY_DIR}/$(LIBPREFIX).so)" >> "$(DESTDIR)$(OPENBLAS_CMAKE_DIR)/$(OPENBLAS_CMAKE_CONFIG)"
@@ -9,7 +9,15 @@ else
 USE_OPENMP = 1
 endif

-
+ifeq ($(CORE), POWER9)
+ifeq ($(USE_OPENMP), 1)
+COMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+FCOMMON_OPT += -O2 -frecursive -mcpu=power9 -mtune=power9 -malign-power -DUSE_OPENMP -fno-fast-math -fopenmp
+else
+COMMON_OPT += -Ofast -mcpu=power9 -mtune=power9 -mvsx -malign-power -fno-fast-math
+FCOMMON_OPT += -O2 -frecursive -mcpu=power9 -mtune=power9 -malign-power -fno-fast-math
+endif
+endif

 ifeq ($(CORE), POWER8)
 ifeq ($(USE_OPENMP), 1)
@@ -21,6 +29,10 @@ FCOMMON_OPT += -O2 -frecursive -mcpu=power8 -mtune=power8 -malign-power -fno-fas
 endif
 endif

+# workaround for C->FORTRAN ABI violation in LAPACKE
+ifeq ($(F_COMPILER), GFORTRAN)
+FCOMMON_OPT += -fno-optimize-sibling-calls
+endif

 FLAMEPATH	= $(HOME)/flame/lib

@@ -3,7 +3,7 @@
 #

 # This library's version
-VERSION = 0.3.4
+VERSION = 0.3.9.dev

 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library
@@ -48,6 +48,8 @@ VERSION = 0.3.4
 # HOSTCC = gcc

 # If you need 32bit binary, define BINARY=32, otherwise define BINARY=64
+# Please note that AVX is not available on 32-bit.
+# Setting BINARY=32 disables AVX/AVX2/AVX-512.
 # BINARY=64

 # About threaded BLAS. It will be automatically detected if you don't
@@ -56,8 +58,14 @@ VERSION = 0.3.4
 # For force setting for multi  threaded, specify USE_THREAD = 1
 # USE_THREAD = 0

+# If you want to build a single-threaded OpenBLAS, but expect to call this
+# from several concurrent threads in some other program, comment this in for
+# thread safety. (This is done automatically for USE_THREAD=1 , and should not
+# be necessary when USE_OPENMP=1)
+# USE_LOCKING = 1
+
 # If you're going to use this library with OpenMP, please comment it in.
-# This flag is always set for POWER8. Don't modify the flag 
+# This flag is always set for POWER8. Don't set USE_OPENMP = 0 if you're targeting POWER8.
 # USE_OPENMP = 1

 # The OpenMP scheduler to use - by default this is "static" and you
@@ -68,36 +76,54 @@ VERSION = 0.3.4
 # allow you to select the scheduler from the environment variable OMP_SCHEDULE
 # CCOMMON_OPT += -DOMP_SCHED=dynamic

-# You can define maximum number of threads. Basically it should be
-# less than actual number of cores. If you don't specify one, it's
-# automatically detected by the the script.
+# You can define the maximum number of threads. Basically it should be less
+# than or equal to the number of CPU threads. If you don't specify one, it's
+# automatically detected by the build system.
+# If SMT (aka. HT) is enabled on the system, it may or may not be beneficial to 
+# restrict NUM_THREADS to the number of physical cores. By default, the automatic 
+# detection includes logical CPUs, thus allowing the use of SMT.
+# Users may opt at runtime to use less than NUM_THREADS threads.
+#
+# Note for package maintainers: you can build OpenBLAS with a large NUM_THREADS
+# value (eg. 32-256) if you expect your users to use that many threads. Due to the way
+# some internal structures are allocated, using a large NUM_THREADS value has a RAM
+# footprint penalty, even if users reduce the actual number of threads at runtime.
 # NUM_THREADS = 24

 # If you have enabled USE_OPENMP and your application would call
-# OpenBLAS's calculation API from multi threads, please comment it in.
-# This flag defines how many instances of OpenBLAS's calculation API can 
-# actually run in parallel. If more threads call OpenBLAS's calculation API,
+# OpenBLAS's calculation API from multiple threads, please comment this in.
+# This flag defines how many instances of OpenBLAS's calculation API can actually
+# run in parallel. If more than NUM_PARALLEL threads call OpenBLAS's calculation API,
 # they need to wait for the preceding API calls to finish or risk data corruption.
 # NUM_PARALLEL = 2

-# if you don't need to install the static library, please comment it in.
+# When multithreading, OpenBLAS needs to use a memory buffer for communicating
+# and collating results for individual subranges of the original matrix. Since
+# the original GotoBLAS of the early 2000s, the default size of this buffer has
+# been set at a value of 32<<20 (which is 32MB) on x86_64 , twice that on PPC.
+# If you expect to handle large problem sizes (beyond about 30000x30000) uncomment
+# this line and adjust the (32<<n) factor if necessary. Usually an insufficient value
+# manifests itself as a crash in the relevant scal kernel (sscal_k, dscal_k etc) 
+# BUFFERSIZE = 25
+
+# If you don't need to install the static library, please comment this in.
 # NO_STATIC = 1

-# if you don't need generate the shared library, please comment it in.
+# If you don't need to generate the shared library, please comment this in.
 # NO_SHARED = 1

-# If you don't need CBLAS interface, please comment it in.
+# If you don't need the CBLAS interface, please comment this in.
 # NO_CBLAS = 1

-# If you only want CBLAS interface without installing Fortran compiler,
-# please comment it in.
+# If you only want the CBLAS interface without installing a Fortran compiler,
+# please comment this in.
 # ONLY_CBLAS = 1

-# If you don't need LAPACK, please comment it in.
-# If you set NO_LAPACK=1, the library automatically sets NO_LAPACKE=1.
+# If you don't need LAPACK, please comment this in.
+# If you set NO_LAPACK=1, the build system automatically sets NO_LAPACKE=1.
 # NO_LAPACK = 1

-# If you don't need LAPACKE (C Interface to LAPACK), please comment it in.
+# If you don't need LAPACKE (C Interface to LAPACK), please comment this in.
 # NO_LAPACKE = 1

 # Build LAPACK Deprecated functions since LAPACK 3.6.0
@@ -106,7 +132,7 @@ BUILD_LAPACK_DEPRECATED = 1
 # Build RecursiveLAPACK on top of LAPACK
 # BUILD_RELAPACK = 1

-# If you want to use legacy threaded Level 3 implementation.
+# If you want to use the legacy threaded Level 3 implementation.
 # USE_SIMPLE_THREADED_LEVEL3 = 1

 # If you want to use the new, still somewhat experimental code that uses
@@ -116,8 +142,8 @@ BUILD_LAPACK_DEPRECATED = 1
 # USE_TLS = 1

 # If you want to drive whole 64bit region by BLAS. Not all Fortran
-# compiler supports this. It's safe to keep comment it out if you
-# are not sure(equivalent to "-i8" option).
+# compilers support this. It's safe to keep this commented out if you
+# are not sure. (This is equivalent to the "-i8" ifort option).
 # INTERFACE64 = 1

 # Unfortunately most of kernel won't give us high quality buffer.
@@ -125,10 +151,18 @@ BUILD_LAPACK_DEPRECATED = 1
 # but it will consume time. If you don't like it, you can disable one.
 NO_WARMUP = 1

-# If you want to disable CPU/Memory affinity on Linux.
+# Comment this in if you want to disable OpenBLAS's CPU/Memory affinity handling.
+# This feature is only implemented on Linux, and is always disabled on other platforms.
+# Enabling affinity handling may improve performance, especially on NUMA systems, but 
+# it may conflict with certain applications that also try to manage affinity.
+# This conflict can result in threads of the application calling OpenBLAS ending up locked
+# to the same core(s) as OpenBLAS, possibly binding all threads to a single core.
+# For this reason, affinity handling is disabled by default. Can be safely enabled if nothing
+# else modifies affinity settings.
+# Note: enabling affinity has been known to cause problems with NumPy and R
 NO_AFFINITY = 1

-# if you are compiling for Linux and you have more than 16 numa nodes or more than 256 cpus
+# If you are compiling for Linux and you have more than 16 numa nodes or more than 256 cpus
 # BIGNUMA = 1

 # Don't use AVX kernel on Sandy Bridge. It is compatible with old compilers
@@ -138,6 +172,10 @@ NO_AFFINITY = 1
 # Don't use Haswell optimizations if binutils is too old (e.g. RHEL6)
 # NO_AVX2 = 1

+# Don't use SkylakeX optimizations if binutils or compiler are too old (the build
+# system will try to determine this automatically)
+# NO_AVX512 = 1
+
 # Don't use parallel make.
 # NO_PARALLEL_MAKE = 1

@@ -162,17 +200,17 @@ NO_AFFINITY = 1
 # time out to improve performance. This number should be from 4 to 30
 # which corresponds to (1 << n) cycles. For example, if you set to 26,
 # thread will be running for (1 << 26) cycles(about 25ms on 3.0GHz
-# system). Also you can control this mumber by THREAD_TIMEOUT
+# system). Also you can control this number by THREAD_TIMEOUT
 # CCOMMON_OPT	+= -DTHREAD_TIMEOUT=26

-# Using special device driver for mapping physically contigous memory
+# Using special device driver for mapping physically contiguous memory
 # to the user space. If bigphysarea is enabled, it will use it.
 # DEVICEDRIVER_ALLOCATION = 1

 # If you need to synchronize FP CSR between threads (for x86/x86_64 only).
 # CONSISTENT_FPCSR = 1

-# If any gemm arguement m, n or k is less or equal this threshold, gemm will be execute
+# If any gemm argument m, n or k is less or equal this threshold, gemm will be execute
 # with single thread. (Actually in recent versions this is a factor proportional to the
 # number of floating point operations necessary for the given problem size, no longer
 # an individual dimension). You can use this setting to avoid the overhead of multi-
@@ -180,7 +218,7 @@ NO_AFFINITY = 1
 # been reported to be optimal for certain workloads (50 is the recommended value for Julia).
 # GEMM_MULTITHREAD_THRESHOLD = 4

-# If you need santy check by comparing reference BLAS. It'll be very
+# If you need sanity check by comparing results to reference BLAS. It'll be very
 # slow (Not implemented yet).
 # SANITY_CHECK = 1

@@ -220,6 +258,21 @@ COMMON_PROF = -pg
 # SYMBOLPREFIX=
 # SYMBOLSUFFIX=

+# Run a C++ based thread safety tester after the build is done.
+# This is mostly intended as a developer feature to spot regressions, but users and
+# package maintainers can enable this if they have doubts about the thread safety of
+# the library, given the configuration in this file.
+# By default, the thread safety tester launches 52 concurrent calculations at the same
+# time.
+#
+# Please note that the test uses ~1300 MiB of RAM for the DGEMM test.
+#
+# The test requires CBLAS to be built, a C++11 capable compiler and the presence of
+# an OpenMP implementation. If you are cross-compiling this test will probably not
+# work at all.
+#
+# CPP_THREAD_SAFETY_TEST = 1
+
 #
 #  End of user configuration
 #
@@ -9,9 +9,24 @@ ifndef TOPDIR
 TOPDIR = .
 endif

+# If ARCH is not set, we use the host system's architecture for getarch compile options.
+ifndef ARCH
+HOSTARCH := $(shell uname -m)
+else
+HOSTARCH = $(ARCH)
+endif
+
 # Catch conflicting usage of ARCH in some BSD environments
 ifeq ($(ARCH), amd64)
 override ARCH=x86_64
+else ifeq ($(ARCH), powerpc64)
+override ARCH=power
+else ifeq ($(ARCH), i386)
+override ARCH=x86
+else ifeq ($(ARCH), aarch64)
+override ARCH=arm64
+else ifeq ($(ARCH), zarch)
+override ARCH=zarch
 endif

 NETLIB_LAPACK_DIR = $(TOPDIR)/lapack-netlib
@@ -59,6 +74,7 @@ endif

 ifdef TARGET
 GETARCH_FLAGS := -DFORCE_$(TARGET)
+GETARCH_FLAGS += -DUSER_TARGET
 endif

 # Force fallbacks for 32bit
@@ -88,6 +104,9 @@ endif
 ifeq ($(TARGET), ZEN)
 GETARCH_FLAGS := -DFORCE_BARCELONA
 endif
+ifeq ($(TARGET), ARMV8)
+GETARCH_FLAGS := -DFORCE_ARMV7
+endif
 endif


@@ -127,7 +146,12 @@ endif
 endif


-
+# On x86_64 build getarch with march=native unless the compiler is PGI. This is required to detect AVX512 support in getarch.
+ifeq ($(HOSTARCH), x86_64)
+ifeq ($(findstring pgcc,$(HOSTCC)),)
+GETARCH_FLAGS += -march=native
+endif
+endif

 ifdef INTERFACE64
 ifneq ($(INTERFACE64), 0)
@@ -145,7 +169,8 @@ GETARCH_FLAGS	+= -DNO_AVX
 endif

 ifeq ($(BINARY), 32)
-GETARCH_FLAGS	+= -DNO_AVX
+GETARCH_FLAGS	+= -DNO_AVX -DNO_AVX2 -DNO_AVX512
+NO_AVX512 = 1
 endif

 ifeq ($(NO_AVX2), 1)
@@ -226,6 +251,10 @@ SMP = 1
 endif
 endif

+ifeq ($(SMP), 1)
+USE_LOCKING = 
+endif
+
 ifndef NEED_PIC
 NEED_PIC = 1
 endif
@@ -242,9 +271,10 @@ OBJCOPY = $(CROSS_SUFFIX)objcopy
 OBJCONV = $(CROSS_SUFFIX)objconv


-# For detect fortran failed, only build BLAS.
+# When fortran support was either not detected or actively deselected, only build BLAS.
 ifeq ($(NOFORTRAN), 1)
 NO_LAPACK = 1
+override FEXTRALIB = 
 endif

 #
@@ -294,12 +324,14 @@ CCOMMON_OPT	+= -DMS_ABI
 endif

 ifeq ($(C_COMPILER), GCC)
-#Test for supporting MS_ABI
+#Version tests for supporting specific features (MS_ABI, POWER9 intrinsics)
 GCCVERSIONGTEQ4 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 4)
 GCCVERSIONGT4 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 4)
+GCCVERSIONGT5 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \> 5)
+GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)
 GCCMINORVERSIONGTEQ7 := $(shell expr `$(CC) -dumpversion | cut -f2 -d.` \>= 7)
 ifeq ($(GCCVERSIONGT4), 1)
-# GCC Majar version > 4
+# GCC Major version > 4
 # It is compatible with MSVC ABI.
 CCOMMON_OPT	+= -DMS_ABI
 endif
@@ -377,6 +409,12 @@ ifneq ($(MAX_STACK_ALLOC), 0)
 CCOMMON_OPT	+= -DMAX_STACK_ALLOC=$(MAX_STACK_ALLOC)
 endif

+ifdef USE_LOCKING
+ifneq ($(USE_LOCKING), 0)
+CCOMMON_OPT	+= -DUSE_LOCKING
+endif
+endif
+
 #
 #  Architecture dependent settings
 #
@@ -512,9 +550,34 @@ endif

 ifeq ($(ARCH), arm64)
 DYNAMIC_CORE =  ARMV8
+DYNAMIC_CORE += CORTEXA53
 DYNAMIC_CORE += CORTEXA57
+DYNAMIC_CORE += CORTEXA72
+DYNAMIC_CORE += CORTEXA73
+DYNAMIC_CORE += FALKOR
 DYNAMIC_CORE += THUNDERX
 DYNAMIC_CORE += THUNDERX2T99
+DYNAMIC_CORE += TSV110
+endif
+
+ifeq ($(ARCH), zarch)
+DYNAMIC_CORE = Z13
+DYNAMIC_CORE += Z14
+endif
+
+ifeq ($(ARCH), power)
+DYNAMIC_CORE = POWER6
+DYNAMIC_CORE += POWER8
+ifneq ($(C_COMPILER), GCC)
+DYNAMIC_CORE += POWER9
+endif
+ifeq ($(C_COMPILER), GCC)
+ifeq ($(GCCVERSIONGT5), 1)
+DYNAMIC_CORE += POWER9
+else
+$(info, OpenBLAS: Your gcc version is too old to build the POWER9 kernels.)
+endif
+endif
 endif

 # If DYNAMIC_CORE is not set, DYNAMIC_ARCH cannot do anything, so force it to empty
@@ -659,7 +722,7 @@ endif

 ifeq ($(C_COMPILER), PGI)
 ifdef BINARY64
-CCOMMON_OPT += -tp p7-64
+CCOMMON_OPT += -tp p7-64 -D__MMX__ -Mnollvm
 else
 CCOMMON_OPT += -tp p7
 endif
@@ -719,6 +782,9 @@ else
 FCOMMON_OPT += -m32
 endif
 endif
+ifneq ($(NO_LAPACKE), 1)
+FCOMMON_OPT += -fno-second-underscore
+endif
 endif
 endif

@@ -727,6 +793,8 @@ CCOMMON_OPT += -DF_INTERFACE_GFORT
 FCOMMON_OPT += -Wall
 # make single-threaded LAPACK calls thread-safe #1847
 FCOMMON_OPT += -frecursive
+# work around ABI problem with passing single-character arguments
+FCOMMON_OPT += -fno-optimize-sibling-calls
 #Don't include -lgfortran, when NO_LAPACK=1 or lsbcc
 ifneq ($(NO_LAPACK), 1)
 EXTRALIB += -lgfortran
@@ -1032,7 +1100,7 @@ ifdef USE_SIMPLE_THREADED_LEVEL3
 CCOMMON_OPT	+= -DUSE_SIMPLE_THREADED_LEVEL3
 endif

-ifdef USE_TLS
+ifeq ($(USE_TLS), 1)
 CCOMMON_OPT += -DUSE_TLS
 endif

@@ -1085,8 +1153,12 @@ endif
 endif

 ifdef NO_AFFINITY
+ifeq ($(NO_AFFINITY), 0)
+override undefine NO_AFFINITY
+else
 CCOMMON_OPT	+= -DNO_AFFINITY
 endif
+endif

 ifdef FUNCTION_PROFILE
 CCOMMON_OPT	+= -DFUNCTION_PROFILE
@@ -1148,8 +1220,6 @@ ifndef FCOMMON_OPT
 FCOMMON_OPT = -O2 -frecursive
 endif

-
-
 override CFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR)
 override PFLAGS     += $(COMMON_OPT) $(CCOMMON_OPT) -I$(TOPDIR) -DPROFILE $(COMMON_PROF)

@@ -1157,6 +1227,12 @@ override FFLAGS     += $(COMMON_OPT) $(FCOMMON_OPT)
 override FPFLAGS    += $(FCOMMON_OPT) $(COMMON_PROF)
 #MAKEOVERRIDES =

+ifdef NEED_PIC
+ifeq (,$(findstring PIC,$(FFLAGS)))
+override FFLAGS += -fPIC
+endif
+endif
+
 #For LAPACK Fortran codes.
 #Disable -fopenmp for LAPACK Fortran codes on Windows.
 ifdef OS_WINDOWS
@@ -9,6 +9,7 @@ endif
 endif

 ifeq ($(CORE), SKYLAKEX)
+ifndef DYNAMIC_ARCH
 ifndef NO_AVX512
 CCOMMON_OPT += -march=skylake-avx512
 FCOMMON_OPT += -march=skylake-avx512
@@ -22,6 +23,22 @@ endif
 endif
 endif
 endif
+endif
+
+ifeq ($(CORE), HASWELL)
+ifndef DYNAMIC_ARCH
+ifndef NO_AVX2
+ifeq ($(C_COMPILER), GCC)
+CCOMMON_OPT += -mavx2
+endif
+ifeq ($(F_COMPILER), GFORTRAN)
+FCOMMON_OPT += -mavx2
+endif
+endif
+endif
+endif
+
+

 ifeq ($(OSNAME), Interix)
 ARFLAGS		= -m x64
@@ -4,3 +4,7 @@ CCOMMON_OPT += -march=z13 -mzvector
 FCOMMON_OPT += -march=z13 -mzvector
 endif

+ifeq ($(CORE), Z14)
+CCOMMON_OPT += -march=z14 -mzvector
+FCOMMON_OPT += -march=z14 -mzvector
+endif
@@ -6,11 +6,13 @@ Travis CI: [![Build Status](https://travis-ci.org/xianyi/OpenBLAS.svg?branch=dev

 AppVeyor: [![Build status](https://ci.appveyor.com/api/projects/status/09sohd35n8nkkx64/branch/develop?svg=true)](https://ci.appveyor.com/project/xianyi/openblas/branch/develop)

+[![Build Status](https://dev.azure.com/xianyi/OpenBLAS/_apis/build/status/xianyi.OpenBLAS?branchName=develop)](https://dev.azure.com/xianyi/OpenBLAS/_build/latest?definitionId=1&branchName=develop)
+
 ## Introduction

 OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.

-Please read the documentation on the OpenBLAS wiki pages: <http://github.com/xianyi/OpenBLAS/wiki>.
+Please read the documentation on the OpenBLAS wiki pages: <https://github.com/xianyi/OpenBLAS/wiki>.

 ## Binary Packages

@@ -22,8 +24,10 @@ You can download them from [file hosting on sourceforge.net](https://sourceforge

 ## Installation from Source

-Download from project homepage, http://xianyi.github.com/OpenBLAS/, or check out the code
+Download from project homepage, https://xianyi.github.com/OpenBLAS/, or check out the code
 using Git from https://github.com/xianyi/OpenBLAS.git.
+Buildtime parameters can be chosen in Makefile.rule, see there for a short description of each option.
+Most can also be given directly on the make or cmake command line.

 ### Dependencies

@@ -63,9 +67,7 @@ A debug version can be built using `make DEBUG=1`.

 ### Compile with MASS support on Power CPU (optional)

-The [IBM MASS](http://www-01.ibm.com/software/awdtools/mass/linux/mass-linux.html) library
-consists of a set of mathematical functions for C, C++, and Fortran applications that are
-are tuned for optimum performance on POWER architectures.
+The [IBM MASS](https://www.ibm.com/support/home/product/W511326D80541V01/other_software/mathematical_acceleration_subsystem) library consists of a set of mathematical functions for C, C++, and Fortran applications that are tuned for optimum performance on POWER architectures.
 OpenBLAS with MASS requires a 64-bit, little-endian OS on POWER.
 The library can be installed as shown:

@@ -101,7 +103,7 @@ The default installation directory is `/opt/OpenBLAS`.

 ## Supported CPUs and Operating Systems

-Please read `GotoBLAS_01Readme.txt`.
+Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by the 2010 GotoBLAS.

 ### Additional supported CPUs

@@ -109,12 +111,13 @@ Please read `GotoBLAS_01Readme.txt`.

 - **Intel Xeon 56xx (Westmere)**: Used GotoBLAS2 Nehalem codes.
 - **Intel Sandy Bridge**: Optimized Level-3 and Level-2 BLAS with AVX on x86-64.
- **Intel Haswell**: Optimized Level-3 and Level-2 BLAS with AVX2 and FMA  on x86-64.
- **Intel Skylake**: Optimized Level-3 and Level-2 BLAS with AVX512 and FMA  on x86-64.
+- **Intel Haswell**: Optimized Level-3 and Level-2 BLAS with AVX2 and FMA on x86-64.
+- **Intel Skylake-X**: Optimized Level-3 and Level-2 BLAS with AVX512 and FMA on x86-64.
 - **AMD Bobcat**: Used GotoBLAS2 Barcelona codes.
 - **AMD Bulldozer**: x86-64 ?GEMM FMA4 kernels. (Thanks to Werner Saar)
 - **AMD PILEDRIVER**: Uses Bulldozer codes with some optimizations.
 - **AMD STEAMROLLER**: Uses Bulldozer codes with some optimizations.
+- **AMD ZEN**: Uses Haswell codes with some optimizations.

 #### MIPS64

@@ -128,26 +131,51 @@ Please read `GotoBLAS_01Readme.txt`.

 #### ARM64

- **ARMv8**: Experimental
- **ARM Cortex-A57**: Experimental
+- **ARMv8**: Basic ARMV8 with small caches, optimized Level-3 and Level-2 BLAS
+- **Cortex-A53**: same as ARMV8 (different cpu specifications)
+- **Cortex A57**: Optimized Level-3 and Level-2 functions
+- **Cortex A72**: same as A57 ( different cpu specifications)
+- **Cortex A73**: same as A57 (different cpu specifications)
+- **Falkor**: same as A57 (different cpu specifications)
+- **ThunderX**: Optimized some Level-1 functions
+- **ThunderX2T99**: Optimized Level-3 BLAS and parts of Levels 1 and 2
+- **TSV110**: Optimized some Level-3 helper functions

 #### PPC/PPC64

- **POWER8**: Optmized Level-3 BLAS and some Level-1, only with `USE_OPENMP=1`
+- **POWER8**: Optimized BLAS, only for PPC64LE (Little Endian), only with `USE_OPENMP=1`
+- **POWER9**: Optimized Level-3 BLAS (real) and some Level-1,2. PPC64LE with OpenMP only. 

 #### IBM zEnterprise System

- **Z13**: Optimized Level-3 BLAS and Level-1,2 (double precision)
+- **Z13**: Optimized Level-3 BLAS and Level-1,2
+- **Z14**: Optimized Level-3 BLAS and (single precision) Level-1,2
+
+### Support for multiple targets in a single library
+
+OpenBLAS can be built for multiple targets with runtime detection of the target cpu by specifiying DYNAMIC_ARCH=1 in Makefile.rule, on the gmake command line or as -DDYNAMIC_ARCH=TRUE in cmake.
+For **x86_64**, the list of targets this activates contains Prescott, Core2, Nehalem, Barcelona, Sandybridge, Bulldozer, Piledriver, Steamroller, Excavator, Haswell, Zen, SkylakeX. For cpu generations not included in this list, the corresponding older model is used. If you also specify DYNAMIC_OLDER=1, specific support for Penryn, Dunnington, Opteron, Opteron/SSE3, Bobcat, Atom and Nano is added. Finally there is an option DYNAMIC_LIST that allows to specify an individual list of targets to include instead of the default.
+DYNAMIC_ARCH is also supported on **x86**, where it translates to Katmai, Coppermine, Northwood, Prescott, Banias,
+Core2, Penryn, Dunnington, Nehalem, Athlon, Opteron, Opteron_SSE3, Barcelona, Bobcat, Atom and Nano.
+On **ARMV8**, it enables support for CortexA53, CortexA57, CortexA72, CortexA73, Falkor, ThunderX, ThunderX2T99, TSV110 as well as generic ARMV8 cpus.
+For **POWER**, the list encompasses POWER6, POWER8 and POWER9, on **ZARCH** it comprises Z13 and Z14.
+The TARGET option can be used in conjunction with DYNAMIC_ARCH=1 to specify which cpu model should be assumed for all the
+common code in the library, usually you will want to set this to the oldest model you expect to encounter.
+Please note that it is not possible to combine support for different architectures, so no combined 32 and 64 bit or x86_64 and arm64 in the same library.

 ### Supported OS

 - **GNU/Linux**
 - **MinGW or Visual Studio (CMake)/Windows**: Please read <https://github.com/xianyi/OpenBLAS/wiki/How-to-use-OpenBLAS-in-Microsoft-Visual-Studio>.
- **Darwin/macOS**: Experimental. Although GotoBLAS2 supports Darwin, we are not macOS experts.
+- **Darwin/macOS/OSX/iOS**: Experimental. Although GotoBLAS2 already supports Darwin, we are not OSX/iOS experts.
 - **FreeBSD**: Supported by the community. We don't actively test the library on this OS.
 - **OpenBSD**: Supported by the community. We don't actively test the library on this OS.
+- **NetBSD**: Supported by the community. We don't actively test the library on this OS.
 - **DragonFly BSD**: Supported by the community. We don't actively test the library on this OS.
 - **Android**: Supported by the community. Please read <https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android>.
+- **AIX**: Supported on PPC up to POWER8
+- **Haiku**: Supported by the community. We don't actively test the library on this OS.
+- **SunOS**: Supported by the community. We don't actively test the library on this OS:

 ## Usage

@@ -201,8 +229,8 @@ Please see Changelog.txt to view the differences between OpenBLAS and GotoBLAS2
 * Please use GCC version 4.6 and above to compile Sandy Bridge AVX kernels on Linux/MinGW/BSD.
 * Please use Clang version 3.1 and above to compile the library on Sandy Bridge microarchitecture.
  Clang 3.0 will generate the wrong AVX binary code.
-* Please use GCC version 6 or LLVM version 6 and above to compile Skyalke AVX512 kernels.
-* The number of CPUs/cores should less than or equal to 256. On Linux `x86_64` (`amd64`),
+* Please use GCC version 6 or LLVM version 6 and above to compile Skylake AVX512 kernels.
+* The number of CPUs/cores should be less than or equal to 256. On Linux `x86_64` (`amd64`),
  there is experimental support for up to 1024 CPUs/cores and 128 numa nodes if you build
  the library with `BIGNUMA=1`.
 * OpenBLAS does not set processor affinity by default.
@@ -48,6 +48,7 @@ POWER5
 POWER6
 POWER7
 POWER8
+POWER9
 PPCG4
 PPC970
 PPC970MP
@@ -90,7 +91,9 @@ CORTEXA73
 FALKOR
 THUNDERX
 THUNDERX2T99
+TSV110

 9.System Z:
 ZARCH_GENERIC
 Z13
+Z14
@@ -35,7 +35,15 @@ environment:
      DYNAMIC_ARCH: ON
      WITH_FORTRAN: no
    - COMPILER: cl
-
+    - COMPILER: MinGW64-gcc-7.2.0-mingw
+      DYNAMIC_ARCH: OFF
+      WITH_FORTRAN: ignore
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      COMPILER: MinGW-gcc-6.3.0-32   
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      COMPILER: MinGW-gcc-5.3.0
+      WITH_FORTRAN: ignore
+    
 install:
  - if [%COMPILER%]==[clang-cl] call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
  - if [%COMPILER%]==[clang-cl] conda config --add channels conda-forge --force
@@ -52,10 +60,17 @@ install:
 before_build:
  - ps: if (-Not (Test-Path .\build)) { mkdir build }
  - cd build
+  - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
+  - if [%COMPILER%]==[MinGW-gcc-5.3.0] set PATH=C:\MinGW\bin;C:\msys64\usr\bin;C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH%
+  - if [%COMPILER%]==[MinGW64-gcc-7.2.0-mingw] set PATH=C:\MinGW\bin;C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH%
+  - if [%COMPILER%]==[MinGW-gcc-6.3.0-32] set PATH=C:\msys64\usr\bin;C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw64\bin;%PATH%
  - if [%COMPILER%]==[cl] cmake -G "Visual Studio 15 2017 Win64" ..
-  - if [%WITH_FORTRAN%]==[no] cmake -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl ..
+  - if [%COMPILER%]==[MinGW64-gcc-7.2.0-mingw] cmake -G "MinGW Makefiles" -DNOFORTRAN=1 ..
+  - if [%COMPILER%]==[MinGW-gcc-6.3.0-32] cmake -G "MSYS Makefiles" -DNOFORTRAN=1 ..
+  - if [%COMPILER%]==[MinGW-gcc-5.3.0] cmake -G "MSYS Makefiles" -DNOFORTRAN=1 ..
+  - if [%WITH_FORTRAN%]==[no] cmake -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DMSVC_STATIC_CRT=ON ..
  - if [%WITH_FORTRAN%]==[yes] cmake -G "Ninja" -DCMAKE_CXX_COMPILER=clang-cl -DCMAKE_C_COMPILER=clang-cl -DCMAKE_Fortran_COMPILER=flang -DBUILD_WITHOUT_LAPACK=no -DNOFORTRAN=0 ..
-  - if [%DYNAMIC_ARCH%]==[ON] cmake -DDYNAMIC_ARCH=ON ..
+  - if [%DYNAMIC_ARCH%]==[ON] cmake -DDYNAMIC_ARCH=ON -DDYNAMIC_LIST='CORE2;NEHALEM;SANDYBRIDGE;BULLDOZER;HASWELL' ..

 build_script:
  - cmake --build .
@@ -64,3 +79,4 @@ test_script:
  - echo Running Test
  - cd utest
  - openblas_utest
+  
@@ -0,0 +1,51 @@
+trigger:
+  # start a new build for every push
+  batch: False
+  branches:
+    include:
+      - develop
+
+jobs:
+# manylinux1 is useful to test because the
+# standard Docker container uses an old version
+# of gcc / glibc
+- job: manylinux1_gcc
+  pool:
+    vmImage: 'ubuntu-16.04'
+  steps:
+  - script: |
+      echo "FROM quay.io/pypa/manylinux1_x86_64
+        COPY . /tmp/openblas
+        RUN cd /tmp/openblas                                      &&  \
+            COMMON_FLAGS='DYNAMIC_ARCH=1 TARGET=NEHALEM NUM_THREADS=32' && \
+            BTYPE='BINARY=64' CC=gcc && \
+            make QUIET_MAKE=1 $COMMON_FLAGS $BTYPE && \
+            make -C test $COMMON_FLAGS $BTYPE && \
+            make -C ctest $COMMON_FLAGS $BTYPE && \
+            make -C utest $COMMON_FLAGS $BTYPE" > Dockerfile
+      docker build .
+    displayName: Run manylinux1 docker build
+- job: Intel_SDE_skx
+  pool:
+    vmImage: 'ubuntu-16.04'
+  steps:
+  - script: |
+      # at the time of writing the available Azure Ubuntu vm image
+      # does not support AVX512VL, so use more recent LTS version
+      echo "FROM ubuntu:bionic
+      COPY . /tmp/openblas
+      RUN apt-get -y update && apt-get -y install \\
+          cmake \\
+          gfortran \\
+          make \\
+          wget
+      RUN mkdir /tmp/SDE && cd /tmp/SDE && \\
+          mkdir sde-external-8.35.0-2019-03-11-lin && \\
+          wget --quiet -O sde-external-8.35.0-2019-03-11-lin.tar.bz2 https://www.dropbox.com/s/fopsnzj67572sj5/sde-external-8.35.0-2019-03-11-lin.tar.bz2?dl=0 && \\
+          tar -xjvf sde-external-8.35.0-2019-03-11-lin.tar.bz2 -C /tmp/SDE/sde-external-8.35.0-2019-03-11-lin --strip-components=1
+      RUN cd /tmp/openblas && CC=gcc make QUIET_MAKE=1 DYNAMIC_ARCH=1 NUM_THREADS=32 BINARY=64
+      CMD cd /tmp/openblas && echo 0 > /proc/sys/kernel/yama/ptrace_scope && CC=gcc OPENBLAS_VERBOSE=2 /tmp/SDE/sde-external-8.35.0-2019-03-11-lin/sde64 -cpuid_in /tmp/SDE/sde-external-8.35.0-2019-03-11-lin/misc/cpuid/skx/cpuid.def -- make -C utest DYNAMIC_ARCH=1 NUM_THREADS=32 BINARY=64" > Dockerfile
+      docker build -t intel_sde .
+      # we need a privileged docker run for sde process attachment
+      docker run --privileged intel_sde
+    displayName: 'Run AVX512 SkylakeX docker build / test'
@@ -207,7 +207,7 @@ int main(int argc, char *argv[]){
  for (i = 0; i < m * n * COMPSIZE; i++) {
    c[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
  }
- 
+
  fprintf(stderr, "          SIZE                   Flops             Time\n");

  for (i = from; i <= to; i += step) {
@@ -197,7 +197,7 @@ int main(int argc, char *argv[]){
   		fprintf(stderr, " %6dx%d : ", (int)m,(int)n);
   		for(j = 0; j < m; j++){
      			for(i = 0; i < n * COMPSIZE; i++){
-				a[i + j * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+				a[j + i * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
      			}
   		}

@@ -208,7 +208,7 @@ int main(int argc, char *argv[]){
 				x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
   			}

-   			for(i = 0; i < n * COMPSIZE * abs(inc_y); i++){
+   			for(i = 0; i < m * COMPSIZE * abs(inc_y); i++){
 				y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
   			}
    			gettimeofday( &start, (struct timezone *)0);
@@ -234,7 +234,7 @@ int main(int argc, char *argv[]){
   		fprintf(stderr, " %6dx%d : ", (int)m,(int)n);
   		for(j = 0; j < m; j++){
      			for(i = 0; i < n * COMPSIZE; i++){
-				a[i + j * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+				a[j + i * m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
      			}
   		}

@@ -245,7 +245,7 @@ int main(int argc, char *argv[]){
 				x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
   			}

-   			for(i = 0; i < n * COMPSIZE * abs(inc_y); i++){
+   			for(i = 0; i < m * COMPSIZE * abs(inc_y); i++){
 				y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
   			}
    			gettimeofday( &start, (struct timezone *)0);
@@ -2,6 +2,8 @@

 argv <- commandArgs(trailingOnly = TRUE)

+if (!is.null(options("matprod")[[1]])) options(matprod = "blas")
+
 nfrom <- 128
 nto <- 2048
 nstep <- 128
@@ -19,7 +21,6 @@ if (length(argv) > 0) {
      loops <- as.numeric(argv[z])
    }
  }
-
 }

 p <- Sys.getenv("OPENBLAS_LOOPS")
@@ -27,29 +28,21 @@ if (p != "") {
  loops <- as.numeric(p)
 }

-
-cat(sprintf(
-  "From %.0f To %.0f Step=%.0f Loops=%.0f\n",
-  nfrom,
-  nto,
-  nstep,
-  loops
-))
+cat(sprintf("From %.0f To %.0f Step=%.0f Loops=%.0f\n", nfrom, nto, nstep, loops))
 cat(sprintf("      SIZE             Flops                   Time\n"))

 n <- nfrom
 while (n <= nto) {
-  A <- matrix(rnorm(n * n), ncol = n, nrow = n)
+  A <- matrix(rnorm(n * n), nrow = n)
  ev <- 0
  z <- system.time(for (l in 1:loops) {
    ev <- eigen(A)
  })

-  mflops <- (26.66 * n * n * n) * loops / (z[3] * 1.0e6)
+  mflops <- (26.66 * n * n * n) * loops / (z[3] * 1e+06)

  st <- sprintf("%.0fx%.0f :", n, n)
  cat(sprintf("%20s %10.2f MFlops %10.6f sec\n", st, mflops, z[3]))

  n <- n + nstep
-
 }
@@ -2,6 +2,8 @@

 argv <- commandArgs(trailingOnly = TRUE)

+if (!is.null(options("matprod")[[1]])) options(matprod = "blas")
+
 nfrom <- 128
 nto <- 2048
 nstep <- 128
@@ -19,7 +21,6 @@ if (length(argv) > 0) {
      loops <- as.numeric(argv[z])
    }
  }
-
 }

 p <- Sys.getenv("OPENBLAS_LOOPS")
@@ -27,26 +28,13 @@ if (p != "") {
  loops <- as.numeric(p)
 }

-
-cat(sprintf(
-  "From %.0f To %.0f Step=%.0f Loops=%.0f\n",
-  nfrom,
-  nto,
-  nstep,
-  loops
-))
+cat(sprintf("From %.0f To %.0f Step=%.0f Loops=%.0f\n", nfrom, nto, nstep, loops))
 cat(sprintf("      SIZE             Flops                   Time\n"))

 n <- nfrom
 while (n <= nto) {
-  A <- matrix(runif(n * n),
-              ncol = n,
-              nrow = n,
-              byrow = TRUE)
-  B <- matrix(runif(n * n),
-              ncol = n,
-              nrow = n,
-              byrow = TRUE)
+  A <- matrix(runif(n * n), nrow = n)
+  B <- matrix(runif(n * n), nrow = n)
  C <- 1

  z <- system.time(for (l in 1:loops) {
@@ -54,11 +42,10 @@ while (n <= nto) {
    l <- l + 1
  })

-  mflops <- (2.0 * n * n * n) * loops / (z[3] * 1.0e6)
+  mflops <- (2.0 * n * n * n) * loops / (z[3] * 1e+06)

  st <- sprintf("%.0fx%.0f :", n, n)
  cat(sprintf("%20s %10.2f MFlops %10.6f sec\n", st, mflops, z[3]))

  n <- n + nstep
-
 }
@@ -2,6 +2,8 @@

 argv <- commandArgs(trailingOnly = TRUE)

+if (!is.null(options("matprod")[[1]])) options(matprod = "blas")
+
 nfrom <- 128
 nto <- 2048
 nstep <- 128
@@ -19,7 +21,6 @@ if (length(argv) > 0) {
      loops <- as.numeric(argv[z])
    }
  }
-
 }

 p <- Sys.getenv("OPENBLAS_LOOPS")
@@ -27,31 +28,22 @@ if (p != "") {
  loops <- as.numeric(p)
 }

-
-cat(sprintf(
-  "From %.0f To %.0f Step=%.0f Loops=%.0f\n",
-  nfrom,
-  nto,
-  nstep,
-  loops
-))
+cat(sprintf("From %.0f To %.0f Step=%.0f Loops=%.0f\n", nfrom, nto, nstep, loops))
 cat(sprintf("      SIZE             Flops                   Time\n"))

 n <- nfrom
 while (n <= nto) {
-  A <- matrix(rnorm(n * n), ncol = n, nrow = n)
-  B <- matrix(rnorm(n * n), ncol = n, nrow = n)
+  A <- matrix(rnorm(n * n), nrow = n)
+  B <- matrix(rnorm(n * n), nrow = n)

  z <- system.time(for (l in 1:loops) {
    solve(A, B)
  })

-  mflops <-
-    (2.0 / 3.0 * n * n * n + 2.0 * n * n * n) * loops / (z[3] * 1.0e6)
+  mflops <- (8.0 / 3 * n * n * n) * loops / (z[3] * 1e+06)

  st <- sprintf("%.0fx%.0f :", n, n)
  cat(sprintf("%20s %10.2f MFlops %10.6f sec\n", st, mflops, z[3]))

  n <- n + nstep
-
 }
@@ -1,7 +1,7 @@
 #!/usr/bin/perl

-use File::Basename;
-use File::Temp qw(tempfile);
+#use File::Basename;
+# use File::Temp qw(tempfile);

 # Checking cross compile
 $hostos   = `uname -s | sed -e s/\-.*//`;    chop($hostos);
@@ -12,7 +12,7 @@ $hostarch = "arm64" if ($hostarch eq "aarch64");
 $hostarch = "power" if ($hostarch =~ /^(powerpc|ppc).*/);
 $hostarch = "zarch" if ($hostarch eq "s390x");

-$tmpf = new File::Temp( UNLINK => 1 );
+#$tmpf = new File::Temp( UNLINK => 1 );
 $binary = $ENV{"BINARY"};

 $makefile = shift(@ARGV);
@@ -31,12 +31,25 @@ if ($?) {

 $cross_suffix = "";

-if (dirname($compiler_name) ne ".") {
-    $cross_suffix .= dirname($compiler_name) . "/";
-}
+eval "use File::Basename";
+if ($@){ 
+    warn "could not load PERL module File::Basename, emulating its functionality";
+    my $dirnam = substr($compiler_name, 0, rindex($compiler_name, "/")-1 );
+    if ($dirnam ne ".") {
+	$cross_suffix .= $dirnam . "/";
+    }
+    my $basnam = substr($compiler_name, rindex($compiler_name,"/")+1, length($compiler_name)-rindex($compiler_name,"/")-1);
+	if ($basnam =~ /([^\s]*-)(.*)/) {
+	$cross_suffix .= $1;
+    }
+} else {
+    if (dirname($compiler_name) ne ".") {
+	$cross_suffix .= dirname($compiler_name) . "/";
+    }

-if (basename($compiler_name) =~ /([^\s]*-)(.*)/) {
-    $cross_suffix .= $1;
+    if (basename($compiler_name) =~ /([^\s]*-)(.*)/) {
+	$cross_suffix .= $1;
+    }
 }

 $compiler = "";
@@ -171,20 +184,26 @@ if ($?) {

 $have_msa = 0;
 if (($architecture eq "mips") || ($architecture eq "mips64")) {
-    $code = '"addvi.b $w0, $w1, 1"';
-    $msa_flags = "-mmsa -mfp64 -msched-weight -mload-store-pairs";
-    print $tmpf "#include <msa.h>\n\n";
-    print $tmpf "void main(void){ __asm__ volatile($code); }\n";
-
-    $args = "$msa_flags -o $tmpf.o -x c $tmpf";
-    my @cmd = ("$compiler_name $args");
-    system(@cmd) == 0;
-    if ($? != 0) {
-        $have_msa = 0;
+    eval "use File::Temp qw(tempfile)";
+    if ($@){ 
+	warn "could not load PERL module File::Temp, so could not check MSA capatibility";
    } else {
-        $have_msa = 1;
+	$tmpf = new File::Temp( SUFFIX => '.c' , UNLINK => 1 );
+	$code = '"addvi.b $w0, $w1, 1"';
+	$msa_flags = "-mmsa -mfp64 -msched-weight -mload-store-pairs";
+	print $tmpf "#include <msa.h>\n\n";
+	print $tmpf "void main(void){ __asm__ volatile($code); }\n";
+
+	$args = "$msa_flags -o $tmpf.o $tmpf";
+	my @cmd = ("$compiler_name $args >/dev/null 2>/dev/null");
+	system(@cmd) == 0;
+	if ($? != 0) {
+	    $have_msa = 0;
+	} else {
+	    $have_msa = 1;
+	}
+	unlink("$tmpf.o");
    }
-    unlink("$tmpf.o");
 }

 $architecture = x86    if ($data =~ /ARCH_X86/);
@@ -204,17 +223,28 @@ $binformat    = bin64  if ($data =~ /BINARY_64/);

 $no_avx512= 0;
 if (($architecture eq "x86") || ($architecture eq "x86_64")) {
-    $code = '"vbroadcastss -4 * 4(%rsi), %zmm2"';
-    print $tmpf "#include <immintrin.h>\n\nint main(void){ __asm__ volatile($code); }\n";
-    $args = " -march=skylake-avx512 -o $tmpf.o -x c $tmpf";
-    my @cmd = ("$compiler_name $args >/dev/null 2>/dev/null");
-    system(@cmd) == 0;
-    if ($? != 0) {
-	$no_avx512 = 1;
-    } else {
+    eval "use File::Temp qw(tempfile)";
+    if ($@){ 
+	warn "could not load PERL module File::Temp, so could not check compiler compatibility with AVX512";
 	$no_avx512 = 0;
+    } else {
+#	$tmpf = new File::Temp( UNLINK => 1 );
+	($fh,$tmpf) = tempfile( SUFFIX => '.c' , UNLINK => 1 );
+	$code = '"vbroadcastss -4 * 4(%rsi), %zmm2"';
+	print $tmpf "#include <immintrin.h>\n\nint main(void){ __asm__ volatile($code); }\n";
+	$args = " -march=skylake-avx512 -c -o $tmpf.o $tmpf";
+	if ($compiler eq "PGI") {
+	    $args = " -tp skylake -c -o $tmpf.o $tmpf";
+	}
+	my @cmd = ("$compiler_name $args >/dev/null 2>/dev/null");
+	system(@cmd) == 0;
+	if ($? != 0) {
+	    $no_avx512 = 1;
+	} else {
+	    $no_avx512 = 0;
+	}
+	unlink("$tmpf.o");
    }
-    unlink("tmpf.o");
 }

 $data = `$compiler_name -S ctest1.c && grep globl ctest1.s | head -n 1 && rm -f ctest1.s`;
@@ -233,6 +263,19 @@ if ($architecture ne $hostarch) {

 $cross = 1 if ($os ne $hostos);

+# rework cross suffix and architecture if we are on OSX cross-compiling for ARMV8-based IOS
+# the initial autodetection will have been confused by the command-line arguments to clang
+# and the cross-compiler apparently still claims to build for x86_64 in its CC -E output
+if (($os eq "Darwin") && ($cross_suffix ne "")) {
+  my $tmpnam = `xcrun --sdk iphoneos --find clang`;
+  $cross_suffix = substr($tmpnam, 0, rindex($tmpnam, "/")+1 ); 
+# this should produce something like $cross_suffix="/Applications/Xcode-10.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/";
+  $cross =1;
+  $architecture = arm64;
+}
+
+
+
 $openmp = "" if $ENV{USE_OPENMP} != 1;

 $linker_L = "";
@@ -278,6 +321,7 @@ $linker_a = "";
 	    && ($flags !~ /kernel32/)
 	    && ($flags !~ /advapi32/)
 	    && ($flags !~ /shell32/)
+	    && ($flags !~ /omp/)
 	    ) {
 	    $linker_l .= $flags . " "
 	}
@@ -73,6 +73,11 @@ double cblas_dasum (OPENBLAS_CONST blasint n, OPENBLAS_CONST double *x, OPENBLAS
 float  cblas_scasum(OPENBLAS_CONST blasint n, OPENBLAS_CONST void  *x, OPENBLAS_CONST blasint incx);
 double cblas_dzasum(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx);

+float  cblas_ssum (OPENBLAS_CONST blasint n, OPENBLAS_CONST float  *x, OPENBLAS_CONST blasint incx);
+double cblas_dsum (OPENBLAS_CONST blasint n, OPENBLAS_CONST double *x, OPENBLAS_CONST blasint incx);
+float  cblas_scsum(OPENBLAS_CONST blasint n, OPENBLAS_CONST void  *x, OPENBLAS_CONST blasint incx);
+double cblas_dzsum(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx);
+
 float  cblas_snrm2 (OPENBLAS_CONST blasint N, OPENBLAS_CONST float  *X, OPENBLAS_CONST blasint incX);
 double cblas_dnrm2 (OPENBLAS_CONST blasint N, OPENBLAS_CONST double *X, OPENBLAS_CONST blasint incX);
 float  cblas_scnrm2(OPENBLAS_CONST blasint N, OPENBLAS_CONST void  *X, OPENBLAS_CONST blasint incX);
@@ -88,6 +93,16 @@ CBLAS_INDEX cblas_idamin(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *x, OPE
 CBLAS_INDEX cblas_icamin(OPENBLAS_CONST blasint n, OPENBLAS_CONST void  *x, OPENBLAS_CONST blasint incx);
 CBLAS_INDEX cblas_izamin(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx);

+CBLAS_INDEX cblas_ismax(OPENBLAS_CONST blasint n, OPENBLAS_CONST float  *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_idmax(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_icmax(OPENBLAS_CONST blasint n, OPENBLAS_CONST void  *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_izmax(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx);
+
+CBLAS_INDEX cblas_ismin(OPENBLAS_CONST blasint n, OPENBLAS_CONST float  *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_idmin(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_icmin(OPENBLAS_CONST blasint n, OPENBLAS_CONST void  *x, OPENBLAS_CONST blasint incx);
+CBLAS_INDEX cblas_izmin(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx);
+
 void cblas_saxpy(OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST float *x, OPENBLAS_CONST blasint incx, float *y, OPENBLAS_CONST blasint incy);
 void cblas_daxpy(OPENBLAS_CONST blasint n, OPENBLAS_CONST double alpha, OPENBLAS_CONST double *x, OPENBLAS_CONST blasint incx, double *y, OPENBLAS_CONST blasint incy);
 void cblas_caxpy(OPENBLAS_CONST blasint n, OPENBLAS_CONST void *alpha, OPENBLAS_CONST void *x, OPENBLAS_CONST blasint incx, void *y, OPENBLAS_CONST blasint incy);
@@ -44,6 +44,14 @@ endif ()


 if (DYNAMIC_ARCH)
+  if (ARM64)
+    set(DYNAMIC_CORE ARMV8 CORTEXA53 CORTEXA57 CORTEXA72 CORTEXA73 FALKOR THUNDERX THUNDERX2T99 TSV110)
+  endif ()
+  
+  if (POWER)
+    set(DYNAMIC_CORE POWER6 POWER8 POWER9)
+  endif ()
+  
  if (X86)
    set(DYNAMIC_CORE KATMAI COPPERMINE NORTHWOOD PRESCOTT BANIAS CORE2 PENRYN DUNNINGTON NEHALEM ATHLON OPTERON OPTERON_SSE3 BARCELONA BOBCAT ATOM NANO)
  endif ()
@@ -69,11 +77,16 @@ if (DYNAMIC_ARCH)
    endif ()
    if (NOT NO_AVX512)
      set(DYNAMIC_CORE ${DYNAMIC_CORE} SKYLAKEX)
+      string(REGEX REPLACE "-march=native" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+   endif ()
+    if (DYNAMIC_LIST)
+	set(DYNAMIC_CORE PRESCOTT ${DYNAMIC_LIST})
    endif ()
  endif ()

  if (NOT DYNAMIC_CORE)
-    unset(DYNAMIC_ARCH)
+    message (STATUS "DYNAMIC_ARCH is not supported on this architecture, removing from options")
+    unset(DYNAMIC_ARCH CACHE)
  endif ()
 endif ()

@@ -3,7 +3,7 @@
 ## Description: Ported from portion of OpenBLAS/Makefile.system
 ##              Sets C related variables.

-if (${CMAKE_C_COMPILER} STREQUAL "GNU" OR ${CMAKE_C_COMPILER} STREQUAL "LSB" OR ${CMAKE_C_COMPILER} STREQUAL "Clang")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_C_COMPILER_ID} STREQUAL "LSB" OR ${CMAKE_C_COMPILER_ID} MATCHES "Clang")

  set(CCOMMON_OPT "${CCOMMON_OPT} -Wall")
  set(COMMON_PROF "${COMMON_PROF} -fno-inline")
@@ -43,7 +43,7 @@ if (${CMAKE_C_COMPILER} STREQUAL "GNU" OR ${CMAKE_C_COMPILER} STREQUAL "LSB" OR
  endif ()
 endif ()

-if (${CMAKE_C_COMPILER} STREQUAL "PGI")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI")
  if (BINARY64)
    set(CCOMMON_OPT "${CCOMMON_OPT} -tp p7-64")
  else ()
@@ -51,7 +51,7 @@ if (${CMAKE_C_COMPILER} STREQUAL "PGI")
  endif ()
 endif ()

-if (${CMAKE_C_COMPILER} STREQUAL "PATHSCALE")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "PATHSCALE")
  if (BINARY64)
    set(CCOMMON_OPT "${CCOMMON_OPT} -m64")
  else ()
@@ -59,7 +59,7 @@ if (${CMAKE_C_COMPILER} STREQUAL "PATHSCALE")
  endif ()
 endif ()

-if (${CMAKE_C_COMPILER} STREQUAL "OPEN64")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "OPEN64")

  if (MIPS64)

@@ -87,7 +87,7 @@ if (${CMAKE_C_COMPILER} STREQUAL "OPEN64")
  endif ()
 endif ()

-if (${CMAKE_C_COMPILER} STREQUAL "SUN")
+if (${CMAKE_C_COMPILER_ID} STREQUAL "SUN")
  set(CCOMMON_OPT "${CCOMMON_OPT} -w")
  if (X86)
    set(CCOMMON_OPT "${CCOMMON_OPT} -m32")
@@ -96,3 +96,10 @@ if (${CMAKE_C_COMPILER} STREQUAL "SUN")
  endif ()
 endif ()

+if (${CORE} STREQUAL "SKYLAKEX")
+  if (NOT DYNAMIC_ARCH)
+    if (NOT NO_AVX512)
+      set (CCOMMON_OPT = "${CCOMMON_OPT} -march=skylake-avx512")
+    endif ()
+  endif ()
+endif ()
@@ -44,7 +44,10 @@ endif ()

 if (${F_COMPILER} STREQUAL "GFORTRAN")
  set(CCOMMON_OPT "${CCOMMON_OPT} -DF_INTERFACE_GFORT")
+  # ensure reentrancy of lapack codes
  set(FCOMMON_OPT "${FCOMMON_OPT} -Wall -frecursive")
+  # work around ABI violation in passing string arguments from C
+  set(FCOMMON_OPT "${FCOMMON_OPT} -fno-optimize-sibling-calls")
  #Don't include -lgfortran, when NO_LAPACK=1 or lsbcc
  if (NOT NO_LAPACK)
    set(EXTRALIB "{EXTRALIB} -lgfortran")
@@ -1,7 +1,7 @@
 # helper functions for the kernel CMakeLists.txt


-# Set the default filenames for L1 objects. Most of these will be overriden by the appropriate KERNEL file.
+# Set the default filenames for L1 objects. Most of these will be overridden by the appropriate KERNEL file.
 macro(SetDefaultL1)
  set(SAMAXKERNEL amax.S)
  set(DAMAXKERNEL amax.S)
@@ -107,6 +107,12 @@ macro(SetDefaultL1)
  set(DAXPBYKERNEL ../arm/axpby.c)
  set(CAXPBYKERNEL ../arm/zaxpby.c)
  set(ZAXPBYKERNEL ../arm/zaxpby.c)
+  set(SSUMKERNEL sum.S)
+  set(DSUMKERNEL sum.S)
+  set(CSUMKERNEL zsum.S)
+  set(ZSUMKERNEL zsum.S)
+  set(QSUMKERNEL sum.S)
+  set(XSUMKERNEL zsum.S)
 endmacro ()

 macro(SetDefaultL2)
@@ -162,4 +168,4 @@ macro(SetDefaultL3)
  set(DGEADD_KERNEL ../generic/geadd.c)
  set(CGEADD_KERNEL ../generic/zgeadd.c)
  set(ZGEADD_KERNEL ../generic/zgeadd.c)
-endmacro ()
+endmacro ()
@@ -115,7 +115,9 @@ set(SLASRC
   stplqt.f stplqt2.f stpmlqt.f
   ssytrd_2stage.f ssytrd_sy2sb.f ssytrd_sb2st.F ssb2st_kernels.f
   ssyevd_2stage.f ssyev_2stage.f ssyevx_2stage.f ssyevr_2stage.f
-   ssbev_2stage.f ssbevx_2stage.f ssbevd_2stage.f ssygv_2stage.f)
+   ssbev_2stage.f ssbevx_2stage.f ssbevd_2stage.f ssygv_2stage.f
+   scombssq.f sgesvdq.f slaorhr_col_getrfnp.f
+   slaorhr_col_getrfnp2.f sorgtsqr.f sorhr_col.f )

 set(SXLASRC sgesvxx.f sgerfsx.f sla_gerfsx_extended.f sla_geamv.f
   sla_gercond.f sla_gerpvgrw.f ssysvxx.f ssyrfsx.f
@@ -210,7 +212,9 @@ set(CLASRC
   ctplqt.f ctplqt2.f ctpmlqt.f
   chetrd_2stage.f chetrd_he2hb.f chetrd_hb2st.F chb2st_kernels.f
   cheevd_2stage.f cheev_2stage.f cheevx_2stage.f cheevr_2stage.f
-   chbev_2stage.f chbevx_2stage.f chbevd_2stage.f chegv_2stage.f)
+   chbev_2stage.f chbevx_2stage.f chbevd_2stage.f chegv_2stage.f
+   cgesvdq.f claunhr_col_getrfnp.f claunhr_col_getrfnp2.f 
+   cungtsqr.f cunhr_col.f )

 set(CXLASRC cgesvxx.f cgerfsx.f cla_gerfsx_extended.f cla_geamv.f
   cla_gercond_c.f cla_gercond_x.f cla_gerpvgrw.f
@@ -299,7 +303,9 @@ set(DLASRC
   dtplqt.f dtplqt2.f dtpmlqt.f
   dsytrd_2stage.f dsytrd_sy2sb.f dsytrd_sb2st.F dsb2st_kernels.f
   dsyevd_2stage.f dsyev_2stage.f dsyevx_2stage.f dsyevr_2stage.f
-   dsbev_2stage.f dsbevx_2stage.f dsbevd_2stage.f dsygv_2stage.f)
+   dsbev_2stage.f dsbevx_2stage.f dsbevd_2stage.f dsygv_2stage.f
+   dcombssq.f dgesvdq.f dlaorhr_col_getrfnp.f
+   dlaorhr_col_getrfnp2.f dorgtsqr.f dorhr_col.f )

 set(DXLASRC dgesvxx.f dgerfsx.f dla_gerfsx_extended.f dla_geamv.f
   dla_gercond.f dla_gerpvgrw.f dsysvxx.f dsyrfsx.f
@@ -398,7 +404,9 @@ set(ZLASRC
   zgelq.f zlaswlq.f zlamswlq.f zgemlq.f
   zhetrd_2stage.f zhetrd_he2hb.f zhetrd_hb2st.F zhb2st_kernels.f
   zheevd_2stage.f zheev_2stage.f zheevx_2stage.f zheevr_2stage.f
-   zhbev_2stage.f zhbevx_2stage.f zhbevd_2stage.f zhegv_2stage.f)
+   zhbev_2stage.f zhbevx_2stage.f zhbevd_2stage.f zhegv_2stage.f
+   zgesvdq.f zlaunhr_col_getrfnp.f zlaunhr_col_getrfnp2.f
+   zungtsqr.f zunhr_col.f)

 set(ZXLASRC zgesvxx.f zgerfsx.f zla_gerfsx_extended.f zla_geamv.f
   zla_gercond_c.f zla_gercond_x.f zla_gerpvgrw.f zsysvxx.f zsyrfsx.f
@@ -715,6 +715,8 @@ set(DSRC
  lapacke_dgesv_work.c
  lapacke_dgesvd.c
  lapacke_dgesvd_work.c
+  lapacke_dgesvdq.c
+  lapacke_dgesvdq_work.c
  lapacke_dgesvdx.c
  lapacke_dgesvdx_work.c
  lapacke_dgesvj.c
@@ -1287,6 +1289,8 @@ set(SSRC
  lapacke_sgesv_work.c
  lapacke_sgesvd.c
  lapacke_sgesvd_work.c
+  lapacke_sgesvdq.c
+  lapacke_sgesvdq_work.c
  lapacke_sgesvdx.c
  lapacke_sgesvdx_work.c
  lapacke_sgesvj.c
@@ -1853,6 +1857,8 @@ set(ZSRC
  lapacke_zgesv_work.c
  lapacke_zgesvd.c
  lapacke_zgesvd_work.c
+  lapacke_zgesvdq.c
+  lapacke_zgesvdq_work.c
  lapacke_zgesvdx.c
  lapacke_zgesvdx_work.c
  lapacke_zgesvj.c
@@ -8,6 +8,11 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
  set(NO_EXPRECISION 1)
 endif ()

+if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD|OpenBSD|NetBSD|DragonFly")
+  set(EXTRALIB "${EXTRALIB} -lm")
+  set(NO_EXPRECISION 1)
+endif ()
+
 if (${CMAKE_SYSTEM_NAME} STREQUAL "AIX")
  set(EXTRALIB "${EXTRALIB} -lm")
 endif ()
@@ -59,6 +59,9 @@ set(FU "")
 if (APPLE OR (MSVC AND NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang"))
  set(FU "_")
 endif()
+if(MINGW AND NOT MINGW64)
+  set(FU "_")
+endif()

 set(COMPILER_ID ${CMAKE_C_COMPILER_ID})
 if (${COMPILER_ID} STREQUAL "GNU")
@@ -82,18 +85,59 @@ endif ()
 # f_check
 if (NOT NOFORTRAN)
  include("${PROJECT_SOURCE_DIR}/cmake/f_check.cmake")
+else ()
+ file(APPEND ${TARGET_CONF_TEMP}
+   "#define BUNDERSCORE _\n"
+   "#define NEEDBUNDERSCORE 1\n")
+ set(BU "_")
 endif ()

 # Cannot run getarch on target if we are cross-compiling
 if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSSTORE"))
  # Write to config as getarch would
+  if (DEFINED TARGET_CORE)
+  set(TCORE ${TARGET_CORE})
+  else()
+  set(TCORE ${CORE})
+  endif()

  # TODO: Set up defines that getarch sets up based on every other target
  # Perhaps this should be inside a different file as it grows larger
  file(APPEND ${TARGET_CONF_TEMP}
-    "#define ${CORE}\n"
-    "#define CHAR_CORENAME \"${CORE}\"\n")
-  if ("${CORE}" STREQUAL "ARMV7")
+    "#define ${TCORE}\n"
+    "#define CORE_${TCORE}\n"
+    "#define CHAR_CORENAME \"${TCORE}\"\n")
+  if ("${TCORE}" STREQUAL "CORE2")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_DATA_SIZE\t32768\n"
+      "#define L1_DATA_LINESIZE\t64\n"
+      "#define L2_SIZE\t1048576\n"
+      "#define L2_LINESIZE\t64\n"
+      "#define DTB_DEFAULT_ENTRIES\t256\n"
+      "#define DTB_SIZE\t4096\n"
+      "#define HAVE_CMOV\n"
+      "#define HAVE_MMX\n"
+      "#define HAVE_SSE\n"
+      "#define HAVE_SSE2\n"
+      "#define HAVE_SSE3\n"
+      "#define HAVE_SSSE3\n"
+      "#define SLOCAL_BUFFER_SIZE\t16384\n"
+      "#define DLOCAL_BUFFER_SIZE\t16384\n"
+      "#define CLOCAL_BUFFER_SIZE\t16384\n"
+      "#define ZLOCAL_BUFFER_SIZE\t16384\n")
+      set(SGEMM_UNROLL_M 8)
+      set(SGEMM_UNROLL_N 4)
+      set(DGEMM_UNROLL_M 4)
+      set(DGEMM_UNROLL_N 4)
+      set(CGEMM_UNROLL_M 4)
+      set(CGEMM_UNROLL_N 2)
+      set(ZGEMM_UNROLL_M 2)
+      set(ZGEMM_UNROLL_N 2)
+      set(CGEMM3M_UNROLL_M 8)
+      set(CGEMM3M_UNROLL_N 4)
+      set(ZGEMM3M_UNROLL_M 4)
+      set(ZGEMM3M_UNROLL_N 4)
+  elseif ("${TCORE}" STREQUAL "ARMV7")
    file(APPEND ${TARGET_CONF_TEMP}
      "#define L1_DATA_SIZE\t65536\n"
      "#define L1_DATA_LINESIZE\t32\n"
@@ -108,7 +152,11 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
    set(SGEMM_UNROLL_N 4)
    set(DGEMM_UNROLL_M 4)
    set(DGEMM_UNROLL_N 4)
-  elseif ("${CORE}" STREQUAL "ARMV8")
+    set(CGEMM_UNROLL_M 2)
+    set(CGEMM_UNROLL_N 2)
+    set(ZGEMM_UNROLL_M 2)
+    set(ZGEMM_UNROLL_N 2)
+  elseif ("${TCORE}" STREQUAL "ARMV8")
    file(APPEND ${TARGET_CONF_TEMP}
      "#define L1_DATA_SIZE\t32768\n"
      "#define L1_DATA_LINESIZE\t64\n"
@@ -116,18 +164,26 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
      "#define L2_LINESIZE\t64\n"
      "#define DTB_DEFAULT_ENTRIES\t64\n"
      "#define DTB_SIZE\t4096\n"
-      "#define L2_ASSOCIATIVE\t32\n")
-    set(SGEMM_UNROLL_M 4)
+      "#define L2_ASSOCIATIVE\t32\n"
+      "#define ARMV8\n")
+    set(SGEMM_UNROLL_M 16)
    set(SGEMM_UNROLL_N 4)
-  elseif ("${CORE}" STREQUAL "CORTEXA57")
+    set(DGEMM_UNROLL_M 8)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 4)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "CORTEXA57" OR "${TCORE}" STREQUAL "CORTEXA53")
    file(APPEND ${TARGET_CONF_TEMP}
-      "#define L1_CODE_SIZE\t49152\n"
+      "#define L1_CODE_SIZE\t32768\n"
      "#define L1_CODE_LINESIZE\t64\n"
      "#define L1_CODE_ASSOCIATIVE\t3\n"
      "#define L1_DATA_SIZE\t32768\n"
      "#define L1_DATA_LINESIZE\t64\n"
      "#define L1_DATA_ASSOCIATIVE\t2\n"
-      "#define L2_SIZE\t2097152\n"
+      "#define L2_SIZE\t262144\n"
      "#define L2_LINESIZE\t64\n"
      "#define L2_ASSOCIATIVE\t16\n"
      "#define DTB_DEFAULT_ENTRIES\t64\n"
@@ -135,15 +191,201 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
      "#define HAVE_VFPV4\n"
      "#define HAVE_VFPV3\n"
      "#define HAVE_VFP\n"
-      "#define HAVE_NEON\n")
+      "#define HAVE_NEON\n"
+      "#define ARMV8\n")
    set(SGEMM_UNROLL_M 16)
    set(SGEMM_UNROLL_N 4)
    set(DGEMM_UNROLL_M 8)
    set(DGEMM_UNROLL_N 4)
    set(CGEMM_UNROLL_M 8)
    set(CGEMM_UNROLL_N 4)
-    set(ZGEMM_UNROLL_M 8)
+    set(ZGEMM_UNROLL_M 4)
    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "CORTEXA72" OR "${TCORE}" STREQUAL "CORTEXA73")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_CODE_SIZE\t49152\n"
+      "#define L1_CODE_LINESIZE\t64\n"
+      "#define L1_CODE_ASSOCIATIVE\t3\n"
+      "#define L1_DATA_SIZE\t32768\n"
+      "#define L1_DATA_LINESIZE\t64\n"
+      "#define L1_DATA_ASSOCIATIVE\t2\n"
+      "#define L2_SIZE\t524288\n"
+      "#define L2_LINESIZE\t64\n"
+      "#define L2_ASSOCIATIVE\t16\n"
+      "#define DTB_DEFAULT_ENTRIES\t64\n"
+      "#define DTB_SIZE\t4096\n"
+      "#define HAVE_VFPV4\n"
+      "#define HAVE_VFPV3\n"
+      "#define HAVE_VFP\n"
+      "#define HAVE_NEON\n"
+      "#define ARMV8\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 8)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 4)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "FALKOR")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_CODE_SIZE\t65536\n"
+      "#define L1_CODE_LINESIZE\t64\n"
+      "#define L1_CODE_ASSOCIATIVE\t3\n"
+      "#define L1_DATA_SIZE\t32768\n"
+      "#define L1_DATA_LINESIZE\t128\n"
+      "#define L1_DATA_ASSOCIATIVE\t2\n"
+      "#define L2_SIZE\t524288\n"
+      "#define L2_LINESIZE\t64\n"
+      "#define L2_ASSOCIATIVE\t16\n"
+      "#define DTB_DEFAULT_ENTRIES\t64\n"
+      "#define DTB_SIZE\t4096\n"
+      "#define HAVE_VFPV4\n"
+      "#define HAVE_VFPV3\n"
+      "#define HAVE_VFP\n"
+      "#define HAVE_NEON\n"
+      "#define ARMV8\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 8)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 4)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "THUNDERX")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_CODE_SIZE\t32768\n"
+      "#define L1_CODE_LINESIZE\t64\n"
+      "#define L1_CODE_ASSOCIATIVE\t3\n"
+      "#define L1_DATA_SIZE\t32768\n"
+      "#define L1_DATA_LINESIZE\t128\n"
+      "#define L1_DATA_ASSOCIATIVE\t2\n"
+      "#define L2_SIZE\t167772164\n"
+      "#define L2_LINESIZE\t128\n"
+      "#define L2_ASSOCIATIVE\t16\n"
+      "#define DTB_DEFAULT_ENTRIES\t64\n"
+      "#define DTB_SIZE\t4096\n"
+      "#define HAVE_VFPV4\n"
+      "#define HAVE_VFPV3\n"
+      "#define HAVE_VFP\n"
+      "#define HAVE_NEON\n"
+      "#define ARMV8\n")
+    set(SGEMM_UNROLL_M 4)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 2)
+    set(DGEMM_UNROLL_N 2)
+    set(CGEMM_UNROLL_M 2)
+    set(CGEMM_UNROLL_N 2)
+    set(ZGEMM_UNROLL_M 2)
+    set(ZGEMM_UNROLL_N 2)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "THUNDERX2T99")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_CODE_SIZE\t32768\n"
+      "#define L1_CODE_LINESIZE\t64\n"
+      "#define L1_CODE_ASSOCIATIVE\t8\n"
+      "#define L1_DATA_SIZE\t32768\n"
+      "#define L1_DATA_LINESIZE\t64\n"
+      "#define L1_DATA_ASSOCIATIVE\t8\n"
+      "#define L2_SIZE\t262144\n"
+      "#define L2_LINESIZE\t64\n"
+      "#define L2_ASSOCIATIVE\t8\n"
+      "#define L3_SIZE\t33554432\n"
+      "#define L3_LINESIZE\t64\n"
+      "#define L3_ASSOCIATIVE\t32\n"
+      "#define DTB_DEFAULT_ENTRIES\t64\n"
+      "#define DTB_SIZE\t4096\n"
+      "#define ARMV8\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 8)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 4)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "TSV110")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define ARMV8\n"
+      "#define L1_CODE_SIZE\t65536\n"
+      "#define L1_CODE_LINESIZE\t64\n"
+      "#define L1_CODE_ASSOCIATIVE\t4\n"
+      "#define L1_DATA_SIZE\t65536\n"
+      "#define L1_DATA_LINESIZE\t64\n"
+      "#define L1_DATA_ASSOCIATIVE\t4\n"
+      "#define L2_SIZE\t524288\n"
+      "#define L2_LINESIZE\t64\n"
+      "#define L2_ASSOCIATIVE\t8\n"
+      "#define DTB_DEFAULT_ENTRIES\t64\n"
+      "#define DTB_SIZE\t4096\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 8)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 4)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 16)
+  elseif ("${TCORE}" STREQUAL "POWER6")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_DATA_SIZE 32768\n"
+      "#define L1_DATA_LINESIZE 128\n"
+      "#define L2_SIZE 524288\n"
+      "#define L2_LINESIZE 128 \n"
+      "#define DTB_DEFAULT_ENTRIES 128\n"
+      "#define DTB_SIZE 4096\n"
+      "#define L2_ASSOCIATIVE 8\n")
+    set(SGEMM_UNROLL_M 4)
+    set(SGEMM_UNROLL_N 4)
+    set(DGEMM_UNROLL_M 4)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 2)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 2)
+    set(ZGEMM_UNROLL_N 4)
+    set(SYMV_P 8)
+  elseif ("${TCORE}" STREQUAL "POWER8")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_DATA_SIZE 32768\n"
+      "#define L1_DATA_LINESIZE 128\n"
+      "#define L2_SIZE 524288\n"
+      "#define L2_LINESIZE 128 \n"
+      "#define DTB_DEFAULT_ENTRIES 128\n"
+      "#define DTB_SIZE 4096\n"
+      "#define L2_ASSOCIATIVE 8\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 8)
+    set(DGEMM_UNROLL_M 16)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 8)
+    set(ZGEMM_UNROLL_N 2)
+    set(SYMV_P 8)
+  elseif ("${TCORE}" STREQUAL "POWER9")
+    file(APPEND ${TARGET_CONF_TEMP}
+      "#define L1_DATA_SIZE 32768\n"
+      "#define L1_DATA_LINESIZE 128\n"
+      "#define L2_SIZE 524288\n"
+      "#define L2_LINESIZE 128 \n"
+      "#define DTB_DEFAULT_ENTRIES 128\n"
+      "#define DTB_SIZE 4096\n"
+      "#define L2_ASSOCIATIVE 8\n")
+    set(SGEMM_UNROLL_M 16)
+    set(SGEMM_UNROLL_N 8)
+    set(DGEMM_UNROLL_M 16)
+    set(DGEMM_UNROLL_N 4)
+    set(CGEMM_UNROLL_M 8)
+    set(CGEMM_UNROLL_N 4)
+    set(ZGEMM_UNROLL_M 8)
+    set(ZGEMM_UNROLL_N 2)
+    set(SYMV_P 8)
  endif()

  # Or should this actually be NUM_CORES?
@@ -163,6 +405,7 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
  file(APPEND ${TARGET_CONF_TEMP}
    "#define GEMM_MULTITHREAD_THRESHOLD\t${GEMM_MULTITHREAD_THRESHOLD}\n")
  # Move to where gen_config_h would place it
+  file(MAKE_DIRECTORY ${TARGET_CONF_DIR})
  file(RENAME ${TARGET_CONF_TEMP} "${TARGET_CONF_DIR}/${TARGET_CONF}")  

 else(NOT CMAKE_CROSSCOMPILING)
@@ -178,6 +421,9 @@ else(NOT CMAKE_CROSSCOMPILING)
    set(GETARCH_FLAGS ${GETARCH_FLAGS} -DFORCE_GENERIC)
  else()
    list(APPEND GETARCH_SRC ${PROJECT_SOURCE_DIR}/cpuid.S)
+    if (DEFINED TARGET_CORE)
+    set(GETARCH_FLAGS ${GETARCH_FLAGS} -DFORCE_${TARGET_CORE})
+  endif ()
  endif ()

  if ("${CMAKE_SYSTEM_NAME}" STREQUAL "WindowsStore")
@@ -39,12 +39,25 @@ if (DEFINED BINARY AND DEFINED TARGET AND BINARY EQUAL 32)
  if (${TARGET} STREQUAL "BULLDOZER" OR ${TARGET} STREQUAL "PILEDRIVER" OR ${TARGET} STREQUAL "ZEN")
    set(TARGET "BARCELONA")
  endif ()
+  if (${TARGET} STREQUAL "ARMV8" OR ${TARGET} STREQUAL "CORTEXA57" OR ${TARGET} STREQUAL "CORTEXA53")
+    set(TARGET "ARMV7")
+  endif ()
 endif ()

 if (DEFINED TARGET)
-if (${TARGET} STREQUAL "SKYLAKEX" AND NOT NO_AVX512)
-  set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512")
-endif()
+  if (${TARGET} STREQUAL "SKYLAKEX" AND NOT NO_AVX512)
+    set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=skylake-avx512")
+  endif()
+  if (${TARGET} STREQUAL "HASWELL" AND NOT NO_AVX2)
+    if (${CMAKE_C_COMPILER_ID} STREQUAL "GNU")
+      execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
+      if (${GCC_VERSION} VERSION_GREATER 4.7 OR ${GCC_VERSION} VERSION_EQUAL 4.7)
+        set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2")
+      endif()
+    elseif (${CMAKE_C_COMPILER_ID} STREQUAL "CLANG")
+      set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2")
+    endif()
+  endif()
 endif()

 if (DEFINED TARGET)
@@ -52,6 +65,18 @@ if (DEFINED TARGET)
  set(GETARCH_FLAGS "-DFORCE_${TARGET}")
 endif ()

+# On x86_64 build getarch with march=native. This is required to detect AVX512 support in getarch.
+if (X86_64 AND NOT ${CMAKE_C_COMPILER_ID} STREQUAL "PGI")
+  set(GETARCH_FLAGS "${GETARCH_FLAGS} -march=native")
+endif ()
+
+# On x86 no AVX support is available
+if (X86 OR X86_64)
+if ((DEFINED BINARY AND BINARY EQUAL 32) OR ("$CMAKE_SIZEOF_VOID_P}" EQUAL "4"))
+  set(GETARCH_FLAGS "${GETARCH_FLAGS} -DNO_AVX -DNO_AVX2 -DNO_AVX512")
+endif ()
+endif ()
+
 if (INTERFACE64)
  message(STATUS "Using 64-bit integers.")
  set(GETARCH_FLAGS	"${GETARCH_FLAGS} -DUSE64BITINT")
@@ -123,10 +148,16 @@ endif ()

 if (USE_THREAD)
  message(STATUS "Multi-threading enabled with ${NUM_THREADS} threads.")
+else()
+  if (${USE_LOCKING})
+    set(CCOMMON_OPT "${CCOMMON_OPT} -DUSE_LOCKING")
+  endif ()
 endif ()

 include("${PROJECT_SOURCE_DIR}/cmake/prebuild.cmake")
-
+if (DEFINED BINARY)
+  message(STATUS "Compiling a ${BINARY}-bit binary.")
+endif ()
 if (NOT DEFINED NEED_PIC)
  set(NEED_PIC 1)
 endif ()
@@ -143,6 +174,9 @@ include("${PROJECT_SOURCE_DIR}/cmake/cc.cmake")
 if (NOT NOFORTRAN)
  # Fortran Compiler dependent settings
  include("${PROJECT_SOURCE_DIR}/cmake/fc.cmake")
+else ()
+set(NO_LAPACK 1)
+set(NO_LAPACKE 1)
 endif ()

 if (BINARY64)
@@ -168,12 +202,24 @@ if (NEED_PIC)
 endif ()

 if (DYNAMIC_ARCH)
-  set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH")
-  if (DYNAMIC_OLDER)
-    set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER")
+  if (X86 OR X86_64 OR ARM64 OR PPC)
+    set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_ARCH")
+    if (DYNAMIC_OLDER)
+      set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_OLDER")
+    endif ()
+  else ()
+    unset (DYNAMIC_ARCH)
+    message (STATUS "DYNAMIC_ARCH is not supported on the target architecture, removing")
  endif ()
 endif ()

+if (DYNAMIC_LIST)
+  set(CCOMMON_OPT "${CCOMMON_OPT} -DDYNAMIC_LIST")
+  foreach(DCORE ${DYNAMIC_LIST})
+    set(CCOMMON_OPT "${CCOMMON_OPT} -DDYN_${DCORE}")
+  endforeach ()
+endif ()
+
 if (NO_LAPACK)
  set(CCOMMON_OPT "${CCOMMON_OPT} -DNO_LAPACK")
  #Disable LAPACK C interface
@@ -243,6 +289,10 @@ set(CCOMMON_OPT "${CCOMMON_OPT} -DMAX_CPU_NUMBER=${NUM_THREADS}")

 set(CCOMMON_OPT "${CCOMMON_OPT} -DMAX_PARALLEL_NUMBER=${NUM_PARALLEL}")

+if (BUFFERSIZE)
+set(CCOMMON_OPT "${CCOMMON_OPT} -DBUFFERSIZE=${BUFFERSIZE}")
+endif ()
+
 if (USE_SIMPLE_THREADED_LEVEL3)
  set(CCOMMON_OPT "${CCOMMON_OPT} -DUSE_SIMPLE_THREADED_LEVEL3")
 endif ()
@@ -263,7 +313,7 @@ endif ()

 set(KERNELDIR	"${PROJECT_SOURCE_DIR}/kernel/${ARCH}")

-# TODO: nead to convert these Makefiles
+# TODO: need to convert these Makefiles
 # include ${PROJECT_SOURCE_DIR}/cmake/${ARCH}.cmake

 if (${CORE} STREQUAL "PPC440")
@@ -15,7 +15,7 @@ if (${HOST_OS} STREQUAL "LINUX")
    EXECUTE_PROCESS( COMMAND uname -o COMMAND tr -d '\n' OUTPUT_VARIABLE OPERATING_SYSTEM)
      if(${OPERATING_SYSTEM} MATCHES "Android")
        set(HOST_OS ANDROID)
-      endif(${OPERATING_SYSTEM} MATCHES "Android")
+      endif()
 endif()


@@ -39,13 +39,45 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc.*|power.*|Power.*")
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "mips64.*")
  set(MIPS64 1)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
-  set(X86_64 1)
+  if (NOT BINARY)
+    if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+      set(X86_64 1)
+    else()
+      set(X86 1)
+    endif()
+  else()
+    if (${BINARY} EQUAL "64")
+       set(X86_64 1)
+    else ()
+       set(X86 1)
+    endif()
+  endif()
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i686.*|i386.*|x86.*|amd64.*|AMD64.*")
  set(X86 1)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm.*|ARM.*)")
  set(ARM 1)
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
-  set(ARM64 1)
+  if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
+    set(ARM64 1)
+  else()
+    set(ARM 1)
+  endif()
+elseif (${CMAKE_CROSSCOMPILING})
+   if (${TARGET} STREQUAL "CORE2")
+    if (NOT BINARY)
+       set(X86 1)
+    elseif (${BINARY} EQUAL "64")
+       set(X86_64 1)
+    else ()
+       set(X86 1)
+    endif()
+   elseif (${TARGET} STREQUAL "ARMV7")
+       set(ARM 1)
+   else()
+       set(ARM64 1)
+   endif ()
+else ()
+   message(WARNING "Target ARCH could not be determined, got \"${CMAKE_SYSTEM_PROCESSOR}\"")
 endif()

 if (X86_64)
@@ -78,10 +110,9 @@ endif()

 if (X86_64 OR X86)
  file(WRITE ${PROJECT_BINARY_DIR}/avx512.tmp "#include <immintrin.h>\n\nint main(void){ __asm__ volatile(\"vbroadcastss -4 * 4(%rsi), %zmm2\"); }")
-execute_process(COMMAND ${CMAKE_C_COMPILER} -march=skylake-avx512 -v -o ${PROJECT_BINARY_DIR}/avx512.o -x c ${PROJECT_BINARY_DIR}/avx512.tmp OUTPUT_QUIET ERROR_QUIET RESULT_VARIABLE NO_AVX512)
+execute_process(COMMAND ${CMAKE_C_COMPILER} -march=skylake-avx512 -c -v -o ${PROJECT_BINARY_DIR}/avx512.o -x c ${PROJECT_BINARY_DIR}/avx512.tmp OUTPUT_QUIET ERROR_QUIET RESULT_VARIABLE NO_AVX512)
 if (NO_AVX512 EQUAL 1)
 set (CCOMMON_OPT "${CCOMMON_OPT} -DNO_AVX512")
 endif()
  file(REMOVE "avx512.tmp" "avx512.o")
 endif()
-
@@ -89,7 +89,7 @@ function(AllCombinations list_in absent_codes_in)
  set(CODES_OUT ${CODES_OUT} PARENT_SCOPE)
 endfunction ()

-# generates object files for each of the sources, using the BLAS naming scheme to pass the funciton name as a preprocessor definition
+# generates object files for each of the sources, using the BLAS naming scheme to pass the function name as a preprocessor definition
 # @param sources_in the source files to build from
 # @param defines_in (optional) preprocessor definitions that will be applied to all objects
 # @param name_in (optional) if this is set this name will be used instead of the filename. Use a * to indicate where the float character should go, if no star the character will be prepended.
@@ -85,6 +85,8 @@ extern "C" {

 #if !defined(_MSC_VER)
 #include <unistd.h>
+#elif _MSC_VER < 1900
+#define snprintf _snprintf
 #endif
 #include <time.h>

@@ -129,7 +131,7 @@ extern "C" {
 #include <time.h>
 #include <unistd.h>
 #include <math.h>
-#ifdef SMP
+#if defined(SMP) || defined(USE_LOCKING)
 #include <pthread.h>
 #endif
 #endif
@@ -198,7 +200,7 @@ extern "C" {
 #error "You can't specify both LOCK operation!"
 #endif

-#ifdef SMP
+#if defined(SMP) || defined(USE_LOCKING)
 #define USE_PTHREAD_LOCK
 #undef	USE_PTHREAD_SPINLOCK
 #endif
@@ -348,6 +350,11 @@ typedef int blasint;
 #endif
 #endif

+#ifdef POWER9
+#ifndef YIELDING
+#define YIELDING        __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
+#endif
+#endif

 /*
 #ifdef PILEDRIVER
@@ -439,7 +446,7 @@ please https://github.com/xianyi/OpenBLAS/issues/246
 typedef char env_var_t[MAX_PATH];
 #define readenv(p, n) 0
 #else
-#ifdef OS_WINDOWS
+#if defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT)
 typedef char env_var_t[MAX_PATH];
 #define readenv(p, n) GetEnvironmentVariable((LPCTSTR)(n), (LPTSTR)(p), sizeof(p))
 #else
@@ -78,7 +78,18 @@ static void __inline blas_lock(volatile BLASULONG *address){

 #define BLAS_LOCK_DEFINED

+#if !defined(OS_DARWIN) && !defined (OS_ANDROID)
+static __inline BLASULONG rpcc(void){
+  BLASULONG ret = 0;
+ 
+  __asm__ __volatile__ ("isb; mrs %0,cntvct_el0":"=r"(ret));

+  return ret;
+}
+
+#define RPCC_DEFINED
+#define RPCC64BIT
+#endif 

 static inline int blas_quickdivide(blasint x, blasint y){
  return x / y;
@@ -103,12 +114,16 @@ static inline int blas_quickdivide(blasint x, blasint y){

 #if defined(ASSEMBLER) && !defined(NEEDPARAM)

-#define PROLOGUE \
-	.text ;\
-	.align	4 ;\
-	.global	REALNAME ;\
-	.type	REALNAME, %function ;\
+.macro PROLOGUE 
+	.text ;
+	.p2align 2 ;
+	.global	REALNAME ;
+#ifndef __APPLE__
+	.type	REALNAME, %function ;
+#endif
 REALNAME:
+.endm
+

 #define EPILOGUE

@@ -19,6 +19,7 @@
 #define	CDOTC_K			cdotc_k
 #define	CNRM2_K			cnrm2_k
 #define	CSCAL_K			cscal_k
+#define	CSUM_K			csum_k
 #define	CSWAP_K			cswap_k
 #define	CROT_K			csrot_k

@@ -249,6 +250,7 @@
 #define	CDOTC_K			gotoblas -> cdotc_k
 #define	CNRM2_K			gotoblas -> cnrm2_k
 #define	CSCAL_K			gotoblas -> cscal_k
+#define	CSUM_K			gotoblas -> csum_k
 #define	CSWAP_K			gotoblas -> cswap_k
 #define	CROT_K			gotoblas -> csrot_k

@@ -19,6 +19,7 @@
 #define	DDOTC_K			ddot_k
 #define	DNRM2_K			dnrm2_k
 #define	DSCAL_K			dscal_k
+#define	DSUM_K			dsum_k
 #define	DSWAP_K			dswap_k
 #define	DROT_K			drot_k

@@ -174,6 +175,7 @@
 #define	DDOTC_K			gotoblas -> ddot_k
 #define	DNRM2_K			gotoblas -> dnrm2_k
 #define	DSCAL_K			gotoblas -> dscal_k
+#define	DSUM_K			gotoblas -> dsum_k
 #define	DSWAP_K			gotoblas -> dswap_k
 #define	DROT_K			gotoblas -> drot_k

@@ -122,6 +122,13 @@ xdouble BLASFUNC(qasum) (blasint *, xdouble *, blasint *);
 double BLASFUNC(dzasum)(blasint *, double *, blasint *);
 xdouble BLASFUNC(qxasum)(blasint *, xdouble *, blasint *);

+FLOATRET  BLASFUNC(ssum) (blasint *, float  *, blasint *);
+FLOATRET  BLASFUNC(scsum)(blasint *, float  *, blasint *);
+double BLASFUNC(dsum) (blasint *, double *, blasint *);
+xdouble BLASFUNC(qsum) (blasint *, xdouble *, blasint *);
+double BLASFUNC(dzsum)(blasint *, double *, blasint *);
+xdouble BLASFUNC(qxsum)(blasint *, xdouble *, blasint *);
+
 blasint    BLASFUNC(isamax)(blasint *, float  *, blasint *);
 blasint    BLASFUNC(idamax)(blasint *, double *, blasint *);
 blasint    BLASFUNC(iqamax)(blasint *, xdouble *, blasint *);
@@ -293,4 +293,150 @@ blasint zlarf_R(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLO
 blasint xlarf_L(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
 blasint xlarf_R(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);

+blasint strtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint dtrtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint qtrtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint ctrtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_URU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_URN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LRU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LRN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ztrtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_URU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_URN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LRU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LRN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint xtrtrs_UNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_URU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_URN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LNU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LNN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LTU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LTN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LRU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LRN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LCU_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LCN_single(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+
+blasint strtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint strtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint dtrtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint dtrtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint qtrtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint qtrtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint ctrtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_URU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_URN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_UCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LRU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LRN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ctrtrs_LCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, float *, float *, BLASLONG);
+blasint ztrtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_URU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_URN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_UCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LRU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LRN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint ztrtrs_LCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, double *, double *, BLASLONG);
+blasint xtrtrs_UNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_URU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_URN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_UCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LNU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LNN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LTU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LTN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LRU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LRN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LCU_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+blasint xtrtrs_LCN_parallel(blas_arg_t *, BLASLONG *, BLASLONG *, xdouble *, xdouble *, BLASLONG);
+
 #endif
@@ -100,6 +100,13 @@ float   casum_k (BLASLONG, float  *, BLASLONG);
 double  zasum_k (BLASLONG, double *, BLASLONG);
 xdouble xasum_k (BLASLONG, xdouble *, BLASLONG);

+float   ssum_k (BLASLONG, float  *, BLASLONG);
+double  dsum_k (BLASLONG, double *, BLASLONG);
+xdouble qsum_k (BLASLONG, xdouble *, BLASLONG);
+float   csum_k (BLASLONG, float  *, BLASLONG);
+double  zsum_k (BLASLONG, double *, BLASLONG);
+xdouble xsum_k (BLASLONG, xdouble *, BLASLONG);
+
 float   samax_k (BLASLONG, float  *, BLASLONG);
 double  damax_k (BLASLONG, double *, BLASLONG);
 xdouble qamax_k (BLASLONG, xdouble *, BLASLONG);
@@ -47,6 +47,14 @@ __global__ void cuda_dgemm_kernel(int, int, int, double *, double *, double *);
 extern "C" {
 #endif

+extern void sgemm_kernel_direct(BLASLONG M, BLASLONG N, BLASLONG K,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB,
+	float * R, BLASLONG strideR);
+
+extern int sgemm_kernel_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K);
+
+
 int sgemm_beta(BLASLONG, BLASLONG, BLASLONG, float,
 	       float  *, BLASLONG, float   *, BLASLONG, float  *, BLASLONG);
 int dgemm_beta(BLASLONG, BLASLONG, BLASLONG, double,
@@ -66,6 +66,7 @@
 #define	DOTC_K			QDOTC_K
 #define	NRM2_K			QNRM2_K
 #define	SCAL_K			QSCAL_K
+#define	SUM_K			QSUM_K
 #define	SWAP_K			QSWAP_K
 #define	ROT_K			QROT_K

@@ -356,6 +357,7 @@
 #define	DOTC_K			DDOTC_K
 #define	NRM2_K			DNRM2_K
 #define	SCAL_K			DSCAL_K
+#define	SUM_K			DSUM_K
 #define	SWAP_K			DSWAP_K
 #define	ROT_K			DROT_K

@@ -639,7 +641,7 @@
 #define IMATCOPY_K_CT		DIMATCOPY_K_CT
 #define IMATCOPY_K_RT		DIMATCOPY_K_RT

-#define GEADD_K                 DGEADD_K 
+#define GEADD_K                 DGEADD_K
 #else

 #define	AMAX_K			SAMAX_K
@@ -658,6 +660,7 @@
 #define	DOTC_K			SDOTC_K
 #define	NRM2_K			SNRM2_K
 #define	SCAL_K			SSCAL_K
+#define	SUM_K			SSUM_K
 #define	SWAP_K			SSWAP_K
 #define	ROT_K			SROT_K

@@ -941,7 +944,7 @@
 #define IMATCOPY_K_CT		SIMATCOPY_K_CT
 #define IMATCOPY_K_RT		SIMATCOPY_K_RT

-#define GEADD_K 		SGEADD_K 
+#define GEADD_K 		SGEADD_K
 #endif
 #else
 #ifdef XDOUBLE
@@ -962,6 +965,7 @@
 #define	DOTC_K			XDOTC_K
 #define	NRM2_K			XNRM2_K
 #define	SCAL_K			XSCAL_K
+#define	SUM_K			XSUM_K
 #define	SWAP_K			XSWAP_K
 #define	ROT_K			XROT_K

@@ -1363,6 +1367,7 @@
 #define	DOTC_K			ZDOTC_K
 #define	NRM2_K			ZNRM2_K
 #define	SCAL_K			ZSCAL_K
+#define	SUM_K			ZSUM_K
 #define	SWAP_K			ZSWAP_K
 #define	ROT_K			ZROT_K

@@ -1765,7 +1770,7 @@
 #define IMATCOPY_K_CTC		ZIMATCOPY_K_CTC
 #define IMATCOPY_K_RTC		ZIMATCOPY_K_RTC

-#define GEADD_K                 ZGEADD_K 
+#define GEADD_K                 ZGEADD_K

 #else

@@ -1785,6 +1790,7 @@
 #define	DOTC_K			CDOTC_K
 #define	NRM2_K			CNRM2_K
 #define	SCAL_K			CSCAL_K
+#define	SUM_K			CSUM_K
 #define	SWAP_K			CSWAP_K
 #define	ROT_K			CROT_K

@@ -2187,7 +2193,7 @@
 #define IMATCOPY_K_CTC		CIMATCOPY_K_CTC
 #define IMATCOPY_K_RTC		CIMATCOPY_K_RTC

-#define GEADD_K                 CGEADD_K 
+#define GEADD_K                 CGEADD_K

 #endif
 #endif
@@ -2800,3 +2806,160 @@ typedef struct {
 #endif

 #endif
+
+#ifndef COMPLEX
+#ifdef XDOUBLE
+#define  TRTRS_UNU_SINGLE  qtrtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  qtrtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  qtrtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  qtrtrs_UTN_single
+#define  TRTRS_LNU_SINGLE  qtrtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  qtrtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  qtrtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  qtrtrs_LTN_single
+#define  TRTRS_UNU_PARALLEL  qtrtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  qtrtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  qtrtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  qtrtrs_UTN_parallel
+#define  TRTRS_LNU_PARALLEL  qtrtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  qtrtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  qtrtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  qtrtrs_LTN_parallel
+
+#elif defined(DOUBLE)
+#define  TRTRS_UNU_SINGLE  dtrtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  dtrtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  dtrtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  dtrtrs_UTN_single
+#define  TRTRS_LNU_SINGLE  dtrtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  dtrtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  dtrtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  dtrtrs_LTN_single
+#define  TRTRS_UNU_PARALLEL  dtrtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  dtrtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  dtrtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  dtrtrs_UTN_parallel
+#define  TRTRS_LNU_PARALLEL  dtrtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  dtrtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  dtrtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  dtrtrs_LTN_parallel
+#else
+#define  TRTRS_UNU_SINGLE  strtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  strtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  strtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  strtrs_UTN_single
+#define  TRTRS_LNU_SINGLE  strtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  strtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  strtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  strtrs_LTN_single
+#define  TRTRS_UNU_PARALLEL  strtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  strtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  strtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  strtrs_UTN_parallel
+#define  TRTRS_LNU_PARALLEL  strtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  strtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  strtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  strtrs_LTN_parallel
+#endif
+#else
+#ifdef XDOUBLE
+#define  TRTRS_UNU_SINGLE  xtrtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  xtrtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  xtrtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  xtrtrs_UTN_single
+#define  TRTRS_URU_SINGLE  xtrtrs_URU_single
+#define  TRTRS_URN_SINGLE  xtrtrs_URN_single
+#define  TRTRS_UCU_SINGLE  xtrtrs_UCU_single
+#define  TRTRS_UCN_SINGLE  xtrtrs_UCN_single
+#define  TRTRS_LNU_SINGLE  xtrtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  xtrtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  xtrtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  xtrtrs_LTN_single
+#define  TRTRS_LRU_SINGLE  xtrtrs_LRU_single
+#define  TRTRS_LRN_SINGLE  xtrtrs_LRN_single
+#define  TRTRS_LCU_SINGLE  xtrtrs_LCU_single
+#define  TRTRS_LCN_SINGLE  xtrtrs_LCN_single
+#define  TRTRS_UNU_PARALLEL  xtrtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  xtrtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  xtrtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  xtrtrs_UTN_parallel
+#define  TRTRS_URU_PARALLEL  xtrtrs_URU_parallel
+#define  TRTRS_URN_PARALLEL  xtrtrs_URN_parallel
+#define  TRTRS_UCU_PARALLEL  xtrtrs_UCU_parallel
+#define  TRTRS_UCN_PARALLEL  xtrtrs_UCN_parallel
+#define  TRTRS_LNU_PARALLEL  xtrtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  xtrtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  xtrtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  xtrtrs_LTN_parallel
+#define  TRTRS_LRU_PARALLEL  xtrtrs_LRU_parallel
+#define  TRTRS_LRN_PARALLEL  xtrtrs_LRN_parallel
+#define  TRTRS_LCU_PARALLEL  xtrtrs_LCU_parallel
+#define  TRTRS_LCN_PARALLEL  xtrtrs_LCN_parallel
+#elif defined(DOUBLE)
+#define  TRTRS_UNU_SINGLE  ztrtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  ztrtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  ztrtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  ztrtrs_UTN_single
+#define  TRTRS_URU_SINGLE  ztrtrs_URU_single
+#define  TRTRS_URN_SINGLE  ztrtrs_URN_single
+#define  TRTRS_UCU_SINGLE  ztrtrs_UCU_single
+#define  TRTRS_UCN_SINGLE  ztrtrs_UCN_single
+#define  TRTRS_LNU_SINGLE  ztrtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  ztrtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  ztrtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  ztrtrs_LTN_single
+#define  TRTRS_LRU_SINGLE  ztrtrs_LRU_single
+#define  TRTRS_LRN_SINGLE  ztrtrs_LRN_single
+#define  TRTRS_LCU_SINGLE  ztrtrs_LCU_single
+#define  TRTRS_LCN_SINGLE  ztrtrs_LCN_single
+#define  TRTRS_UNU_PARALLEL  ztrtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  ztrtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  ztrtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  ztrtrs_UTN_parallel
+#define  TRTRS_URU_PARALLEL  ztrtrs_URU_parallel
+#define  TRTRS_URN_PARALLEL  ztrtrs_URN_parallel
+#define  TRTRS_UCU_PARALLEL  ztrtrs_UCU_parallel
+#define  TRTRS_UCN_PARALLEL  ztrtrs_UCN_parallel
+#define  TRTRS_LNU_PARALLEL  ztrtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  ztrtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  ztrtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  ztrtrs_LTN_parallel
+#define  TRTRS_LRU_PARALLEL  ztrtrs_LRU_parallel
+#define  TRTRS_LRN_PARALLEL  ztrtrs_LRN_parallel
+#define  TRTRS_LCU_PARALLEL  ztrtrs_LCU_parallel
+#define  TRTRS_LCN_PARALLEL  ztrtrs_LCN_parallel
+#else
+#define  TRTRS_UNU_SINGLE  ctrtrs_UNU_single
+#define  TRTRS_UNN_SINGLE  ctrtrs_UNN_single
+#define  TRTRS_UTU_SINGLE  ctrtrs_UTU_single
+#define  TRTRS_UTN_SINGLE  ctrtrs_UTN_single
+#define  TRTRS_URU_SINGLE  ctrtrs_URU_single
+#define  TRTRS_URN_SINGLE  ctrtrs_URN_single
+#define  TRTRS_UCU_SINGLE  ctrtrs_UCU_single
+#define  TRTRS_UCN_SINGLE  ctrtrs_UCN_single
+#define  TRTRS_LNU_SINGLE  ctrtrs_LNU_single
+#define  TRTRS_LNN_SINGLE  ctrtrs_LNN_single
+#define  TRTRS_LTU_SINGLE  ctrtrs_LTU_single
+#define  TRTRS_LTN_SINGLE  ctrtrs_LTN_single
+#define  TRTRS_LRU_SINGLE  ctrtrs_LRU_single
+#define  TRTRS_LRN_SINGLE  ctrtrs_LRN_single
+#define  TRTRS_LCU_SINGLE  ctrtrs_LCU_single
+#define  TRTRS_LCN_SINGLE  ctrtrs_LCN_single
+#define  TRTRS_UNU_PARALLEL  ctrtrs_UNU_parallel
+#define  TRTRS_UNN_PARALLEL  ctrtrs_UNN_parallel
+#define  TRTRS_UTU_PARALLEL  ctrtrs_UTU_parallel
+#define  TRTRS_UTN_PARALLEL  ctrtrs_UTN_parallel
+#define  TRTRS_URU_PARALLEL  ctrtrs_URU_parallel
+#define  TRTRS_URN_PARALLEL  ctrtrs_URN_parallel
+#define  TRTRS_UCU_PARALLEL  ctrtrs_UCU_parallel
+#define  TRTRS_UCN_PARALLEL  ctrtrs_UCN_parallel
+#define  TRTRS_LNU_PARALLEL  ctrtrs_LNU_parallel
+#define  TRTRS_LNN_PARALLEL  ctrtrs_LNN_parallel
+#define  TRTRS_LTU_PARALLEL  ctrtrs_LTU_parallel
+#define  TRTRS_LTN_PARALLEL  ctrtrs_LTN_parallel
+#define  TRTRS_LRU_PARALLEL  ctrtrs_LRU_parallel
+#define  TRTRS_LRN_PARALLEL  ctrtrs_LRN_parallel
+#define  TRTRS_LCU_PARALLEL  ctrtrs_LCU_parallel
+#define  TRTRS_LCN_PARALLEL  ctrtrs_LCN_parallel
+#endif
+#endif
@@ -63,6 +63,7 @@ BLASLONG (*ismin_k) (BLASLONG, float *, BLASLONG);

  float  (*snrm2_k) (BLASLONG, float *, BLASLONG);
  float  (*sasum_k) (BLASLONG, float *, BLASLONG);
+  float  (*ssum_k)  (BLASLONG, float *, BLASLONG);
  int    (*scopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG);
  float  (*sdot_k)  (BLASLONG, float *, BLASLONG, float *, BLASLONG);
  double (*dsdot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG);
@@ -154,6 +155,7 @@ BLASLONG (*idmin_k) (BLASLONG, double *, BLASLONG);

  double (*dnrm2_k) (BLASLONG, double *, BLASLONG);
  double (*dasum_k) (BLASLONG, double *, BLASLONG);
+  double (*dsum_k)  (BLASLONG, double *, BLASLONG);
  int    (*dcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG);
  double (*ddot_k)  (BLASLONG, double *, BLASLONG, double *, BLASLONG);
  int    (*drot_k)  (BLASLONG, double *, BLASLONG, double *, BLASLONG, double, double);
@@ -245,6 +247,7 @@ BLASLONG (*iqmin_k) (BLASLONG, xdouble *, BLASLONG);

 xdouble (*qnrm2_k) (BLASLONG, xdouble *, BLASLONG);
 xdouble (*qasum_k) (BLASLONG, xdouble *, BLASLONG);
+ xdouble (*qsum_k)  (BLASLONG, xdouble *, BLASLONG);
  int    (*qcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG);
 xdouble (*qdot_k)  (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG);
  int    (*qrot_k)  (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG, xdouble, xdouble);
@@ -332,6 +335,7 @@ BLASLONG (*icamin_k)(BLASLONG, float *, BLASLONG);

  float (*cnrm2_k) (BLASLONG, float *, BLASLONG);
  float (*casum_k) (BLASLONG, float *, BLASLONG);
+  float (*csum_k)  (BLASLONG, float *, BLASLONG);
  int    (*ccopy_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG);
  openblas_complex_float (*cdotu_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG);
  openblas_complex_float (*cdotc_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG);
@@ -495,6 +499,7 @@ BLASLONG (*izamin_k)(BLASLONG, double *, BLASLONG);

  double (*znrm2_k) (BLASLONG, double *, BLASLONG);
  double (*zasum_k) (BLASLONG, double *, BLASLONG);
+  double (*zsum_k)  (BLASLONG, double *, BLASLONG);
  int    (*zcopy_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG);
  openblas_complex_double (*zdotu_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG);
  openblas_complex_double (*zdotc_k) (BLASLONG, double *, BLASLONG, double *, BLASLONG);
@@ -660,6 +665,7 @@ BLASLONG (*ixamin_k)(BLASLONG, xdouble *, BLASLONG);

  xdouble (*xnrm2_k) (BLASLONG, xdouble *, BLASLONG);
  xdouble (*xasum_k) (BLASLONG, xdouble *, BLASLONG);
+  xdouble (*xsum_k) (BLASLONG, xdouble *, BLASLONG);
  int    (*xcopy_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG);
  openblas_complex_xdouble (*xdotu_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG);
  openblas_complex_xdouble (*xdotc_k) (BLASLONG, xdouble *, BLASLONG, xdouble *, BLASLONG);
@@ -39,7 +39,36 @@
 #ifndef COMMON_POWER
 #define COMMON_POWER

-#if defined(POWER8)
+#define str(x)	#x
+
+#ifdef OS_AIX
+#define XXSPLTD(T,A,z)	xxpermdi	T, A, A, 0b##z##z
+#define XXMRGHD(T,A,B)	xxpermdi	T, A, B, 0b00
+#define XXMRGLD(T,A,B)	xxpermdi	T, A, B, 0b11
+#define XXSWAPD(T,A)	xxpermdi	T, A, A, 0b10
+#define XVMOVDP(T,A)	xvcpsgndp	T, A, A
+
+#define XXSPLTD_S(T,A,z)	"xxpermdi	" str(T) ", " str(A) ", " str(A) ", 0b" str(z ## z) "	\n\t"
+#define XXMRGHD_S(T,A,B)	"xxpermdi	" str(T) ", " str(A) ", " str(B) ", 0b00	\n\t"
+#define XXMRGLD_S(T,A,B)	"xxpermdi	" str(T) ", " str(A) ", " str(B) ", 0b11	\n\t"
+#define XXSWAPD_S(T,A)	"xxpermdi	" str(T) ", " str(A) ", " str(A) ", 0b10	\n\t"
+
+#else
+#define XXSPLTD(T,A,z)	xxspltd	T, A, z
+#define XXMRGHD(T,A,B)	xxmrghd	T, A, B
+#define XXMRGLD(T,A,B)	xxmrgld	T, A, B
+#define XXSWAPD(T,A)	xxswapd	T, A
+#define XVMOVDP(T,A)	xvmovdp	T, A
+
+#define XXSPLTD_S(T,A,z)	"xxspltd	" str(T) ", " str(A) ", " str(z)"	\n\t"
+#define XXMRGHD_S(T,A,B)	"xxmrghd	" str(T) ", " str(A) ", " str(B)"	\n\t"
+#define XXMRGLD_S(T,A,B)	"xxmrgld	" str(T) ", " str(A) ", " str(B)"	\n\t"
+#define XXSWAPD_S(T,A)	"xxswapd	" str(T) ", " str(A) "	\n\t"
+
+#endif
+
+
+#if defined(POWER8) || defined(POWER9)
 #define MB		__asm__ __volatile__ ("eieio":::"memory")
 #define WMB		__asm__ __volatile__ ("eieio":::"memory")
 #else
@@ -241,7 +270,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define HAVE_PREFETCH
 #endif

-#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8)
+#if defined(POWER3) || defined(POWER6) || defined(PPCG4) || defined(CELL) || defined(POWER8) || defined(POWER9) || defined(PPC970)
 #define DCBT_ARG	0
 #else
 #define DCBT_ARG	8
@@ -263,7 +292,7 @@ static inline int blas_quickdivide(blasint x, blasint y){
 #define L1_PREFETCH	dcbtst
 #endif

-#if defined(POWER8)
+#if defined(POWER8) || defined(POWER9)
 #define L1_DUALFETCH
 #define L1_PREFETCHSIZE (16 + 128 * 100)
 #define L1_PREFETCH	dcbtst
@@ -499,7 +528,7 @@ static inline int blas_quickdivide(blasint x, blasint y){

 #if defined(ASSEMBLER) && !defined(NEEDPARAM)

-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
 #ifndef __64BIT__
 #define PROLOGUE \
 	.section .text;\
@@ -598,9 +627,14 @@ REALNAME:;\
 #ifndef __64BIT__
 #define PROLOGUE \
 	.machine "any";\
+	.toc;\
 	.globl .REALNAME;\
+	.globl REALNAME;\
+	.csect REALNAME[DS],3;\
+REALNAME:;\
+	.long .REALNAME, TOC[tc0], 0;\
 	.csect .text[PR],5;\
-.REALNAME:;
+.REALNAME:

 #define EPILOGUE \
 _section_.text:;\
@@ -611,9 +645,14 @@ _section_.text:;\

 #define PROLOGUE \
 	.machine "any";\
+	.toc;\
 	.globl .REALNAME;\
+	.globl REALNAME;\
+	.csect REALNAME[DS],3;\
+REALNAME:;\
+	.llong .REALNAME, TOC[tc0], 0;\
 	.csect .text[PR], 5;\
-.REALNAME:;
+.REALNAME:

 #define EPILOGUE \
 _section_.text:;\
@@ -774,7 +813,7 @@ Lmcount$lazy_ptr:

 #define HALT		mfspr	r0, 1023

-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
 #if defined(PPC440) || defined(PPC440FP2)
 #undef  MAX_CPU_NUMBER
 #define MAX_CPU_NUMBER 1
@@ -802,7 +841,7 @@ Lmcount$lazy_ptr:
 #define BUFFER_SIZE     (  2 << 20)
 #elif defined(PPC440FP2)
 #define BUFFER_SIZE     ( 16 << 20)
-#elif defined(POWER8)
+#elif defined(POWER8) || defined(POWER9)
 #define BUFFER_SIZE     ( 64 << 20)
 #else
 #define BUFFER_SIZE     ( 16 << 20)
@@ -819,7 +858,7 @@ Lmcount$lazy_ptr:
 #define MAP_ANONYMOUS MAP_ANON
 #endif

-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
 #ifndef __64BIT__
 #define FRAMESLOT(X) (((X) * 4) + 8)
 #else
@@ -19,6 +19,7 @@
 #define	QDOTC_K			qdot_k
 #define	QNRM2_K			qnrm2_k
 #define	QSCAL_K			qscal_k
+#define	QSUM_K			qsum_k
 #define	QSWAP_K			qswap_k
 #define	QROT_K			qrot_k

@@ -161,6 +162,7 @@
 #define	QDOTC_K			gotoblas -> qdot_k
 #define	QNRM2_K			gotoblas -> qnrm2_k
 #define	QSCAL_K			gotoblas -> qscal_k
+#define	QSUM_K			gotoblas -> qsum_k
 #define	QSWAP_K			gotoblas -> qswap_k
 #define	QROT_K			gotoblas -> qrot_k

@@ -12,6 +12,7 @@
 #define	ISMAX_K			ismax_k
 #define	ISMIN_K			ismin_k
 #define	SASUM_K			sasum_k
+#define	SSUM_K			ssum_k
 #define	SAXPYU_K		saxpy_k
 #define	SAXPYC_K		saxpy_k
 #define	SCOPY_K			scopy_k
@@ -170,6 +171,7 @@
 #define	ISMAX_K			gotoblas -> ismax_k
 #define	ISMIN_K			gotoblas -> ismin_k
 #define	SASUM_K			gotoblas -> sasum_k
+#define	SSUM_K			gotoblas -> ssum_k
 #define	SAXPYU_K		gotoblas -> saxpy_k
 #define	SAXPYC_K		gotoblas -> saxpy_k
 #define	SCOPY_K			gotoblas -> scopy_k
@@ -45,7 +45,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * SIZE must be carefully chosen to be:
 * - as small as possible to maximize the number of stack allocation
 * - large enough to support all architectures and kernel
- * Chosing a too small SIZE will lead to a stack smashing.
+ * Choosing a SIZE too small will lead to a stack smashing.
 */
 #define STACK_ALLOC(SIZE, TYPE, BUFFER)                                        \
  /* make it volatile because some function (ex: dgemv_n.S) */                 \
@@ -194,10 +194,6 @@ int trsm_thread(int mode, BLASLONG m, BLASLONG n,

 int syrk_thread(int mode, blas_arg_t *, BLASLONG *, BLASLONG *, int (*function)(), void *, void *, BLASLONG);

-int beta_thread(int mode, BLASLONG m, BLASLONG n,
-		double alpha_r, double alpha_i,
-		void *c, BLASLONG ldc, int (*fuction)());
-
 int getrf_thread(int mode, BLASLONG m, BLASLONG n, BLASLONG k,
 		 void *offsetA, BLASLONG lda,
 		 void *offsetB, BLASLONG jb,
@@ -19,6 +19,7 @@
 #define	XDOTC_K			xdotc_k
 #define	XNRM2_K			xnrm2_k
 #define	XSCAL_K			xscal_k
+#define	XSUM_K			xsum_k
 #define	XSWAP_K			xswap_k
 #define	XROT_K			xqrot_k

@@ -227,6 +228,7 @@
 #define	XDOTC_K			gotoblas -> xdotc_k
 #define	XNRM2_K			gotoblas -> xnrm2_k
 #define	XSCAL_K			gotoblas -> xscal_k
+#define	XSUM_K			gotoblas -> xsum_k
 #define	XSWAP_K			gotoblas -> xswap_k
 #define	XROT_K			gotoblas -> xqrot_k

@@ -187,7 +187,7 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 	
  y = blas_quick_divide_table[y];

-  __asm__ __volatile__  ("mull %0" :"=d" (result) :"a"(x), "0" (y));
+  __asm__ __volatile__  ("mull %0" :"=d" (result), "+a"(x): "0" (y));

  return result;
 #endif
@@ -214,7 +214,7 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 #endif

 #if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
-//Enable some optimazation for barcelona.
+//Enable some optimization for barcelona.
 #define BARCELONA_OPTIMIZATION
 #endif

@@ -129,7 +129,8 @@ static __inline void cpuid(int op, int *eax, int *ebx, int *ecx, int *edx){
  *ecx=cpuinfo[2];
  *edx=cpuinfo[3];
 #else
-        __asm__ __volatile__("cpuid"
+        __asm__ __volatile__("mov $0, %%ecx;"
+			     "cpuid"
 			     : "=a" (*eax),
 			     "=b" (*ebx),
 			     "=c" (*ecx),
@@ -210,7 +211,7 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 	
  y = blas_quick_divide_table[y];

-  __asm__ __volatile__  ("mull %0" :"=d" (result) :"a"(x), "0" (y));
+  __asm__ __volatile__  ("mull %0" :"=d" (result), "+a"(x) : "0" (y));

  return result;
 }
@@ -224,7 +225,11 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 #endif
 #define HUGE_PAGESIZE	( 2 << 20)

+#ifndef BUFFERSIZE
 #define BUFFER_SIZE	(32 << 20)
+#else
+#define BUFFER_SIZE	(32 << BUFFERSIZE)
+#endif

 #define SEEK_ADDRESS

@@ -276,7 +281,7 @@ static __inline int blas_quickdivide(unsigned int x, unsigned int y){
 #ifdef ASSEMBLER

 #if defined(PILEDRIVER) || defined(BULLDOZER) || defined(STEAMROLLER) || defined(EXCAVATOR)
-//Enable some optimazation for barcelona.
+//Enable some optimization for barcelona.
 #define BARCELONA_OPTIMIZATION
 #endif

@@ -19,6 +19,7 @@
 #define	ZDOTC_K			zdotc_k
 #define	ZNRM2_K			znrm2_k
 #define	ZSCAL_K			zscal_k
+#define	ZSUM_K			zsum_k
 #define	ZSWAP_K			zswap_k
 #define	ZROT_K			zdrot_k

@@ -249,6 +250,7 @@
 #define	ZDOTC_K			gotoblas -> zdotc_k
 #define	ZNRM2_K			gotoblas -> znrm2_k
 #define	ZSCAL_K			gotoblas -> zscal_k
+#define	ZSUM_K			gotoblas -> zsum_k
 #define	ZSWAP_K			gotoblas -> zswap_k
 #define	ZROT_K			gotoblas -> zdrot_k

@@ -0,0 +1,14 @@
+include ../Makefile.rule
+
+all :: dgemv_tester dgemm_tester
+
+dgemv_tester :
+	$(CXX) $(COMMON_OPT) -Wall -Wextra -Wshadow -fopenmp -std=c++11 dgemv_thread_safety.cpp ../libopenblas.a -lpthread -o dgemv_tester
+	./dgemv_tester
+
+dgemm_tester : dgemv_tester
+	$(CXX) $(COMMON_OPT) -Wall -Wextra -Wshadow -fopenmp -std=c++11 dgemm_thread_safety.cpp ../libopenblas.a -lpthread -o dgemm_tester
+	./dgemm_tester
+
+clean ::
+	rm -f dgemv_tester dgemm_tester
@@ -0,0 +1,55 @@
+inline void pauser(){
+    /// a portable way to pause a program
+    std::string dummy;
+    std::cout << "Press enter to continue...";
+    std::getline(std::cin, dummy);
+}
+
+void FillMatrices(std::vector<std::vector<double>>& matBlock, std::mt19937_64& PRNG, std::uniform_real_distribution<double>& rngdist, const blasint randomMatSize, const uint32_t numConcurrentThreads, const uint32_t numMat){
+	for(uint32_t i=0; i<numMat; i++){
+		for(uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize*randomMatSize); j++){
+			matBlock[i][j] = rngdist(PRNG);
+		}
+	}
+	for(uint32_t i=numMat; i<(numConcurrentThreads*numMat); i+=numMat){
+		for(uint32_t j=0; j<numMat; j++){
+			matBlock[i+j] = matBlock[j];
+		}
+	}
+}
+
+void FillVectors(std::vector<std::vector<double>>& vecBlock, std::mt19937_64& PRNG, std::uniform_real_distribution<double>& rngdist, const blasint randomMatSize, const uint32_t numConcurrentThreads, const uint32_t numVec){
+	for(uint32_t i=0; i<numVec; i++){
+		for(uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize); j++){
+			vecBlock[i][j] = rngdist(PRNG);
+		}
+	}
+	for(uint32_t i=numVec; i<(numConcurrentThreads*numVec); i+=numVec){
+		for(uint32_t j=0; j<numVec; j++){
+			vecBlock[i+j] = vecBlock[j];
+		}
+	}
+}
+
+std::mt19937_64 InitPRNG(){
+	std::random_device rd;
+	std::mt19937_64 PRNG(rd()); //seed PRNG using /dev/urandom or similar OS provided RNG
+	std::uniform_real_distribution<double> rngdist{-1.0, 1.0};
+	//make sure the internal state of the PRNG is properly mixed by generating 10M random numbers
+	//PRNGs often have unreliable distribution uniformity and other statistical properties before their internal state is sufficiently mixed
+	for (uint32_t i=0;i<10000000;i++) rngdist(PRNG);
+	return PRNG;
+}
+
+void PrintMatrices(const std::vector<std::vector<double>>& matBlock, const blasint randomMatSize, const uint32_t numConcurrentThreads, const uint32_t numMat){
+	for (uint32_t i=0;i<numConcurrentThreads*numMat;i++){
+		std::cout<<i<<std::endl;
+		for (uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize); j++){
+			for (uint32_t k = 0; k < static_cast<uint32_t>(randomMatSize); k++){
+				std::cout<<matBlock[i][j*randomMatSize + k]<<"  ";
+			}
+			std::cout<<std::endl;
+		}
+		std::cout<<std::endl;
+	}
+}
@@ -0,0 +1,92 @@
+#include <iostream>
+#include <vector>
+#include <random>
+#include <future>
+#include <omp.h>
+#include "../cblas.h"
+#include "cpp_thread_safety_common.h"
+
+void launch_cblas_dgemm(double* A, double* B, double* C, const blasint randomMatSize){
+	cblas_dgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, randomMatSize, randomMatSize, randomMatSize, 1.0, A, randomMatSize, B, randomMatSize, 0.1, C, randomMatSize);
+}
+
+int main(int argc, char* argv[]){
+	blasint randomMatSize = 1024; //dimension of the random square matrices used
+	uint32_t numConcurrentThreads = 52; //number of concurrent calls of the functions being tested
+	uint32_t numTestRounds = 16; //number of testing rounds before success exit
+	
+	if (argc > 4){
+		std::cout<<"ERROR: too many arguments for thread safety tester"<<std::endl;
+		abort();
+	}
+	
+	if(argc == 4){
+		std::vector<std::string> cliArgs;
+		for (int i = 1; i < argc; i++){
+			cliArgs.push_back(argv[i]);
+			std::cout<<argv[i]<<std::endl;
+		}
+		randomMatSize = std::stoul(cliArgs[0]);
+		numConcurrentThreads = std::stoul(cliArgs[1]);
+		numTestRounds = std::stoul(cliArgs[2]);
+	}
+	
+	std::uniform_real_distribution<double> rngdist{-1.0, 1.0};
+	std::vector<std::vector<double>> matBlock(numConcurrentThreads*3);
+	std::vector<std::future<void>> futureBlock(numConcurrentThreads);
+	
+	std::cout<<"*----------------------------*\n";
+	std::cout<<"| DGEMM thread safety tester |\n";
+	std::cout<<"*----------------------------*\n";
+	std::cout<<"Size of random matrices(N=M=K): "<<randomMatSize<<'\n';
+	std::cout<<"Number of concurrent calls into OpenBLAS : "<<numConcurrentThreads<<'\n';
+	std::cout<<"Number of testing rounds : "<<numTestRounds<<'\n';
+	std::cout<<"This test will need "<<(static_cast<uint64_t>(randomMatSize*randomMatSize)*numConcurrentThreads*3*8)/static_cast<double>(1024*1024)<<" MiB of RAM\n"<<std::endl;
+	
+	std::cout<<"Initializing random number generator..."<<std::flush;
+	std::mt19937_64 PRNG = InitPRNG();
+	std::cout<<"done\n";
+	
+	std::cout<<"Preparing to test CBLAS DGEMM thread safety\n";
+	std::cout<<"Allocating matrices..."<<std::flush;
+	for(uint32_t i=0; i<(numConcurrentThreads*3); i++){
+		matBlock[i].resize(randomMatSize*randomMatSize);
+	}
+	std::cout<<"done\n";
+	//pauser();
+	std::cout<<"Filling matrices with random numbers..."<<std::flush;
+	FillMatrices(matBlock, PRNG, rngdist, randomMatSize, numConcurrentThreads, 3);
+	//PrintMatrices(matBlock, randomMatSize, numConcurrentThreads, 3);
+	std::cout<<"done\n";
+	std::cout<<"Testing CBLAS DGEMM thread safety\n";
+	omp_set_num_threads(numConcurrentThreads);
+	for(uint32_t R=0; R<numTestRounds; R++){
+		std::cout<<"DGEMM round #"<<R<<std::endl;
+		std::cout<<"Launching "<<numConcurrentThreads<<" threads simultaneously using OpenMP..."<<std::flush;
+		#pragma omp parallel for default(none) shared(futureBlock, matBlock, randomMatSize, numConcurrentThreads)
+		for(uint32_t i=0; i<numConcurrentThreads; i++){
+			futureBlock[i] = std::async(std::launch::async, launch_cblas_dgemm, &matBlock[i*3][0], &matBlock[i*3+1][0], &matBlock[i*3+2][0], randomMatSize);
+			//launch_cblas_dgemm( &matBlock[i][0], &matBlock[i+1][0], &matBlock[i+2][0]);
+		}
+		std::cout<<"done\n";
+		std::cout<<"Waiting for threads to finish..."<<std::flush;
+		for(uint32_t i=0; i<numConcurrentThreads; i++){
+			futureBlock[i].get();
+		}
+		std::cout<<"done\n";
+		//PrintMatrices(matBlock, randomMatSize, numConcurrentThreads, 3);
+		std::cout<<"Comparing results from different threads..."<<std::flush;
+		for(uint32_t i=3; i<(numConcurrentThreads*3); i+=3){ //i is the index of matrix A, for a given thread
+			for(uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize*randomMatSize); j++){
+				if (std::abs(matBlock[i+2][j] - matBlock[2][j]) > 1.0E-13){ //i+2 is the index of matrix C, for a given thread
+					std::cout<<"ERROR: one of the threads returned a different result! Index : "<<i+2<<std::endl;
+					std::cout<<"CBLAS DGEMM thread safety test FAILED!"<<std::endl;
+					return -1;
+				}
+			}
+		}
+		std::cout<<"OK!\n"<<std::endl;
+	}
+	std::cout<<"CBLAS DGEMM thread safety test PASSED!\n"<<std::endl;
+	return 0;
+}
@@ -0,0 +1,101 @@
+#include <iostream>
+#include <vector>
+#include <random>
+#include <future>
+#include <omp.h>
+#include "../cblas.h"
+#include "cpp_thread_safety_common.h"
+
+void launch_cblas_dgemv(double* A, double* x, double* y, const blasint randomMatSize){
+	const blasint inc = 1;
+	cblas_dgemv(CblasColMajor, CblasNoTrans, randomMatSize, randomMatSize, 1.0, A, randomMatSize, x, inc, 0.1, y, inc);
+}
+
+int main(int argc, char* argv[]){
+	blasint randomMatSize = 1024; //dimension of the random square matrices and vectors being used
+	uint32_t numConcurrentThreads = 52; //number of concurrent calls of the functions being tested
+	uint32_t numTestRounds = 16; //number of testing rounds before success exit
+	
+	if (argc > 4){
+		std::cout<<"ERROR: too many arguments for thread safety tester"<<std::endl;
+		abort();
+	}
+	if(argc == 4){
+		std::vector<std::string> cliArgs;
+		for (int i = 1; i < argc; i++){
+			cliArgs.push_back(argv[i]);
+			std::cout<<argv[i]<<std::endl;
+		}
+		randomMatSize = std::stoul(cliArgs.at(0));
+		numConcurrentThreads = std::stoul(cliArgs.at(1));
+		numTestRounds = std::stoul(cliArgs.at(2));
+	}
+	
+	std::uniform_real_distribution<double> rngdist{-1.0, 1.0};
+	std::vector<std::vector<double>> matBlock(numConcurrentThreads);
+	std::vector<std::vector<double>> vecBlock(numConcurrentThreads*2);
+	std::vector<std::future<void>> futureBlock(numConcurrentThreads);
+	
+	std::cout<<"*----------------------------*\n";
+	std::cout<<"| DGEMV thread safety tester |\n";
+	std::cout<<"*----------------------------*\n";
+	std::cout<<"Size of random matrices and vectors(N=M): "<<randomMatSize<<'\n';
+	std::cout<<"Number of concurrent calls into OpenBLAS : "<<numConcurrentThreads<<'\n';
+	std::cout<<"Number of testing rounds : "<<numTestRounds<<'\n';
+	std::cout<<"This test will need "<<((static_cast<uint64_t>(randomMatSize*randomMatSize)*numConcurrentThreads*8)+(static_cast<uint64_t>(randomMatSize)*numConcurrentThreads*8*2))/static_cast<double>(1024*1024)<<" MiB of RAM\n"<<std::endl;
+	
+	std::cout<<"Initializing random number generator..."<<std::flush;
+	std::mt19937_64 PRNG = InitPRNG();
+	std::cout<<"done\n";
+	
+	std::cout<<"Preparing to test CBLAS DGEMV thread safety\n";
+	std::cout<<"Allocating matrices..."<<std::flush;
+	for(uint32_t i=0; i<numConcurrentThreads; i++){
+		matBlock.at(i).resize(randomMatSize*randomMatSize);
+	}
+	std::cout<<"done\n";
+	std::cout<<"Allocating vectors..."<<std::flush;
+	for(uint32_t i=0; i<(numConcurrentThreads*2); i++){
+		vecBlock.at(i).resize(randomMatSize);
+	}
+	std::cout<<"done\n";
+	//pauser();
+	
+	std::cout<<"Filling matrices with random numbers..."<<std::flush;
+	FillMatrices(matBlock, PRNG, rngdist, randomMatSize, numConcurrentThreads, 1);
+	//PrintMatrices(matBlock, randomMatSize, numConcurrentThreads);
+	std::cout<<"done\n";
+	std::cout<<"Filling vectors with random numbers..."<<std::flush;
+	FillVectors(vecBlock, PRNG, rngdist, randomMatSize, numConcurrentThreads, 2);
+	std::cout<<"done\n";
+	
+	std::cout<<"Testing CBLAS DGEMV thread safety"<<std::endl;
+	omp_set_num_threads(numConcurrentThreads);
+	for(uint32_t R=0; R<numTestRounds; R++){
+		std::cout<<"DGEMV round #"<<R<<std::endl;
+		std::cout<<"Launching "<<numConcurrentThreads<<" threads simultaneously using OpenMP..."<<std::flush;
+		#pragma omp parallel for default(none) shared(futureBlock, matBlock, vecBlock, randomMatSize, numConcurrentThreads)
+		for(uint32_t i=0; i<numConcurrentThreads; i++){
+			futureBlock[i] = std::async(std::launch::async, launch_cblas_dgemv, &matBlock[i][0], &vecBlock[i*2][0], &vecBlock[i*2+1][0], randomMatSize);
+		}
+		std::cout<<"done\n";
+		std::cout<<"Waiting for threads to finish..."<<std::flush;
+		for(uint32_t i=0; i<numConcurrentThreads; i++){
+			futureBlock[i].get();
+		}
+		std::cout<<"done\n";
+		std::cout<<"Comparing results from different threads..."<<std::flush;
+		for(uint32_t i=2; i<(numConcurrentThreads*2); i+=2){ //i is the index of vector x, for a given thread
+			for(uint32_t j = 0; j < static_cast<uint32_t>(randomMatSize); j++){
+				if (std::abs(vecBlock[i+1][j] - vecBlock[1][j]) > 1.0E-13){ //i+1 is the index of vector y, for a given thread
+					std::cout<<"ERROR: one of the threads returned a different result! Index : "<<i+1<<std::endl;
+					std::cout<<"CBLAS DGEMV thread safety test FAILED!"<<std::endl;
+					return -1;
+				}
+			}
+		}
+		std::cout<<"OK!\n"<<std::endl;
+	}
+	std::cout<<"CBLAS DGEMV thread safety test PASSED!\n"<<std::endl;
+	return 0;
+}
@@ -53,6 +53,7 @@
 #define VENDOR_SIS	  8
 #define VENDOR_TRANSMETA  9
 #define VENDOR_NSC	 10
+#define VENDOR_HYGON	 11
 #define VENDOR_UNKNOWN   99

 #define BITMASK(a, b, c) ((((a) >> (b)) & (c)))
@@ -116,6 +117,7 @@
 #define CORE_EXCAVATOR   26
 #define CORE_ZEN         27
 #define CORE_SKYLAKEX    28
+#define CORE_DHYANA	 29

 #define HAVE_SSE      (1 <<  0)
 #define HAVE_SSE2     (1 <<  1)
@@ -139,6 +141,7 @@
 #define HAVE_FMA4     (1 <<  19)
 #define HAVE_FMA3     (1 <<  20)
 #define HAVE_AVX512VL (1 <<  21)
+#define HAVE_AVX2     (1 <<  22)

 #define CACHE_INFO_L1_I     1
 #define CACHE_INFO_L1_D     2
@@ -214,5 +217,8 @@ typedef struct {
 #define CPUTYPE_EXCAVATOR 		50
 #define CPUTYPE_ZEN 			51
 #define CPUTYPE_SKYLAKEX		52
+#define CPUTYPE_DHYANA			53
+
+#define CPUTYPE_HYGON_UNKNOWN		54

 #endif
@@ -34,7 +34,7 @@
 #define CPU_CORTEXA15       	4

 static char *cpuname[] = {
-  "UNKOWN",
+  "UNKNOWN",
  "ARMV6",
  "ARMV7",
  "CORTEXA9",
@@ -39,6 +39,8 @@
 // Cavium
 #define CPU_THUNDERX      7
 #define CPU_THUNDERX2T99  8
+//Hisilicon
+#define CPU_TSV110        9

 static char *cpuname[] = {
  "UNKNOWN",
@@ -49,7 +51,8 @@ static char *cpuname[] = {
  "CORTEXA73",
  "FALKOR",
  "THUNDERX",
-  "THUNDERX2T99"
+  "THUNDERX2T99",
+  "TSV110"
 };

 static char *cpuname_lower[] = {
@@ -61,7 +64,8 @@ static char *cpuname_lower[] = {
  "cortexa73",
  "falkor",
  "thunderx",
-  "thunderx2t99"
+  "thunderx2t99",
+  "tsv110"
 };

 int get_feature(char *search)
@@ -90,7 +94,7 @@ int get_feature(char *search)
 	if( p == NULL ) return 0;

 	t = strtok(p," ");
-	while( t = strtok(NULL," "))
+	while( (t = strtok(NULL," ")))
 	{
 		if (!strcmp(t, search))   { return(1); }
 	}
@@ -145,6 +149,9 @@ int detect(void)
 			return CPU_THUNDERX;
    else if (strstr(cpu_implementer, "0x43") && strstr(cpu_part, "0x0af"))
 			return CPU_THUNDERX2T99;
+    // HiSilicon
+    else if (strstr(cpu_implementer, "0x48") && strstr(cpu_part, "0xd01"))
+                        return CPU_TSV110;
 	}

 	p = (char *) NULL ;
@@ -199,6 +206,33 @@ void get_subdirname(void)
 	printf("arm64");
 }

+void get_cpucount(void)
+{
+int n=0;
+
+#ifdef linux
+	FILE *infile;
+  	char buffer[2048], *p,*t;
+  	p = (char *) NULL ;
+
+  	infile = fopen("/proc/cpuinfo", "r");
+
+	while (fgets(buffer, sizeof(buffer), infile))
+	{
+
+		if (!strncmp("processor", buffer, 9))
+		n++;
+  	}
+
+  	fclose(infile);
+
+	printf("#define NUM_CORES %d\n",n);
+#endif
+
+}
+
+
+
 void get_cpuconfig(void)
 {

@@ -270,7 +304,7 @@ void get_cpuconfig(void)
 			break;

 		case CPU_THUNDERX2T99:
-			printf("#define VULCAN                        \n");
+			printf("#define THUNDERX2T99                  \n");
 			printf("#define L1_CODE_SIZE         32768    \n");
 			printf("#define L1_CODE_LINESIZE     64       \n");
 			printf("#define L1_CODE_ASSOCIATIVE  8        \n");
@@ -286,7 +320,23 @@ void get_cpuconfig(void)
 			printf("#define DTB_DEFAULT_ENTRIES  64       \n");
 			printf("#define DTB_SIZE             4096     \n");
 			break;
+			
+		case CPU_TSV110:
+			printf("#define TSV110                        \n");
+			printf("#define L1_CODE_SIZE         65536    \n");
+			printf("#define L1_CODE_LINESIZE     64       \n");
+			printf("#define L1_CODE_ASSOCIATIVE  4        \n");
+			printf("#define L1_DATA_SIZE         65536    \n");
+			printf("#define L1_DATA_LINESIZE     64       \n");
+			printf("#define L1_DATA_ASSOCIATIVE  4        \n");
+			printf("#define L2_SIZE              524228   \n");
+			printf("#define L2_LINESIZE          64       \n");
+			printf("#define L2_ASSOCIATIVE       8        \n");
+			printf("#define DTB_DEFAULT_ENTRIES  64       \n");
+			printf("#define DTB_SIZE             4096     \n");
+			break;	
 	}
+	get_cpucount();
 }


@@ -322,12 +372,10 @@ void get_features(void)
 	if( p == NULL ) return;

 	t = strtok(p," ");
-	while( t = strtok(NULL," "))
+	while( (t = strtok(NULL," ")))
 	{
 	}

 #endif
 	return;
 }
-
-
@@ -75,7 +75,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CPU_1004K	2

 static char *cpuname[] = {
-  "UNKOWN",
+  "UNKNOWN",
  "P5600",
  "1004K"
 };
@@ -79,7 +79,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CPU_I6500       6

 static char *cpuname[] = {
-  "UNKOWN",
+  "UNKNOWN",
  "SICORTEX",
  "LOONGSON3A",
  "LOONGSON3B",
@@ -94,7 +94,7 @@ char *corename[] = {
  "CELL",
  "PPCG4",
  "POWER8",
-  "POWER8"   	
+  "POWER9"   	
 };

 int detect(void){
@@ -124,7 +124,7 @@ int detect(void){
  if (!strncasecmp(p, "POWER6", 6)) return CPUTYPE_POWER6;
  if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
  if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
-  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER8;
+  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
  if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
  if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;

@@ -136,7 +136,7 @@ int detect(void){
  char buffer[512], *p;

  p = (char *)NULL;
-  infile = popen("prtconf|grep 'Processor Type'");
+  infile = popen("prtconf|grep 'Processor Type'", "r");
  while (fgets(buffer, sizeof(buffer), infile)){
    if (!strncmp("Pro", buffer, 3)){
 	p = strchr(buffer, ':') + 2;
@@ -156,7 +156,7 @@ int detect(void){
  if (!strncasecmp(p, "POWER6", 6)) return CPUTYPE_POWER6;
  if (!strncasecmp(p, "POWER7", 6)) return CPUTYPE_POWER6;
  if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
-  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER8;
+  if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
  if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
  if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
  return CPUTYPE_POWER5;
@@ -180,7 +180,7 @@ int id;
 __asm __volatile("mfpvr %0" : "=r"(id));
 switch ( id >> 16 ) {
  case 0x4e: // POWER9
-    return CPUTYPE_POWER8;
+    return CPUTYPE_POWER9;
    break;
  case 0x4d:
  case 0x4b: // POWER8/8E 
@@ -97,10 +97,10 @@ static C_INLINE void cpuid(int op, int *eax, int *ebx, int *ecx, int *edx){
    ("mov %%ebx, %%edi;"
     "cpuid;"
     "xchgl %%ebx, %%edi;"
-     : "=a" (*eax), "=D" (*ebx), "=c" (*ecx), "=d" (*edx) : "a" (op) : "cc");
+     : "=a" (*eax), "=D" (*ebx), "=c" (*ecx), "=d" (*edx) : "a" (op), "c" (0) : "cc");
 #else
  __asm__ __volatile__
-    ("cpuid": "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "a" (op) : "cc");
+    ("cpuid": "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "a" (op) , "c" (0) : "cc");
 #endif
 }

@@ -211,6 +211,44 @@ int support_avx(){
 #endif
 }

+int support_avx2(){
+#ifndef NO_AVX2
+  int eax, ebx, ecx=0, edx;
+  int ret=0;
+
+  if (!support_avx()) 
+    return 0;
+  cpuid(7, &eax, &ebx, &ecx, &edx);
+  if((ebx & (1<<7)) != 0)
+      ret=1;  //OS supports AVX2
+  return ret;
+#else
+  return 0;
+#endif
+}
+
+int support_avx512(){
+#if !defined(NO_AVX) && !defined(NO_AVX512)
+  int eax, ebx, ecx, edx;
+  int ret=0;
+
+  if (!support_avx()) 
+    return 0;
+  cpuid(7, &eax, &ebx, &ecx, &edx);
+  if((ebx & 32) != 32){
+      ret=0;  //OS does not even support AVX2
+  }
+  if((ebx & (1<<31)) != 0){
+    xgetbv(0, &eax, &edx); 
+    if((eax & 0xe0) == 0xe0)
+      ret=1;  //OS supports AVX512VL
+  }
+  return ret;
+#else
+  return 0;
+#endif
+}
+

 int get_vendor(void){
  int eax, ebx, ecx, edx;
@@ -233,6 +271,7 @@ int get_vendor(void){
  if (!strcmp(vendor, " SiS SiS SiS")) return VENDOR_SIS;
  if (!strcmp(vendor, "GenuineTMx86")) return VENDOR_TRANSMETA;
  if (!strcmp(vendor, "Geode by NSC")) return VENDOR_NSC;
+  if (!strcmp(vendor, "HygonGenuine")) return VENDOR_HYGON;

  if ((eax == 0) || ((eax & 0x500) != 0)) return VENDOR_INTEL;

@@ -294,6 +333,8 @@ int get_cputype(int gettype){
    if ((ecx & (1 << 20)) != 0) feature |= HAVE_SSE4_2;
 #ifndef NO_AVX
    if (support_avx()) feature |= HAVE_AVX;
+    if (support_avx2()) feature |= HAVE_AVX2;
+    if (support_avx512()) feature |= HAVE_AVX512VL;
    if ((ecx & (1 << 12)) != 0) feature |= HAVE_FMA3;
 #endif

@@ -1006,7 +1047,9 @@ int get_cacheinfo(int type, cache_info_t *cacheinfo){
    }
  }

-  if ((get_vendor() == VENDOR_AMD) || (get_vendor() == VENDOR_CENTAUR)) {
+  if ((get_vendor() == VENDOR_AMD) ||
+      (get_vendor() == VENDOR_HYGON) ||
+      (get_vendor() == VENDOR_CENTAUR)) {
    cpuid(0x80000005, &eax, &ebx, &ecx, &edx);

    LDTB.size        = 4096;
@@ -1154,7 +1197,11 @@ int get_cpuname(void){
 	case  3:
 	case  5:
 	case  6:
+#if defined(__x86_64__) || defined(__amd64__)
+	  return CPUTYPE_CORE2;
+#else
 	  return CPUTYPE_PENTIUM2;
+#endif
 	case  7:
 	case  8:
 	case 10:
@@ -1168,7 +1215,7 @@ int get_cpuname(void){
 	  return CPUTYPE_CORE2;
 	}
 	break;
-      case 1:
+      case 1:  // family 6 exmodel 1
 	switch (model) {
 	case  6:
 	  return CPUTYPE_CORE2;
@@ -1185,7 +1232,7 @@ int get_cpuname(void){
 	  return CPUTYPE_DUNNINGTON;
 	}
 	break;
-      case  2:
+      case  2: // family 6 exmodel 2
 	switch (model) {
 	case 5:
 	  //Intel Core (Clarkdale) / Core (Arrandale)
@@ -1214,7 +1261,7 @@ int get_cpuname(void){
 	  return CPUTYPE_NEHALEM;
 	}
 	break;
-      case 3:
+      case 3: // family 6 exmodel 3
 	switch (model) {
 	case  7:
 	    // Bay Trail	
@@ -1228,57 +1275,47 @@ int get_cpuname(void){
 	    return CPUTYPE_NEHALEM;
        case 12:
 	case 15:
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 13:
 	  //Broadwell
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	}
 	break;
-      case 4:
+      case 4: // family 6 exmodel 4
        switch (model) {
        case 5:
 	case 6:
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 7:
 	case 15:
 	  //Broadwell
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 14:
 	  //Skylake
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 12:
@@ -1288,84 +1325,85 @@ int get_cpuname(void){
 	    return CPUTYPE_NEHALEM;
        }
        break;
-      case 5:
+      case 5:  // family 6 exmodel 5
        switch (model) {
 	case 6:
 	  //Broadwell
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 5:
 	  // Skylake X
-#ifndef NO_AVX512
-	  return CPUTYPE_SKYLAKEX;
-#else
-	  if(support_avx())
-#ifndef NO_AVX2
-	  return CPUTYPE_HASWELL;
-#else
-	  return CPUTYPE_SANDYBRIDGE;
-#endif
+          if(support_avx512())
+            return CPUTYPE_SKYLAKEX;
+          if(support_avx2())
+            return CPUTYPE_HASWELL;
+          if(support_avx())
+	    return CPUTYPE_SANDYBRIDGE;
 	  else
 	  return CPUTYPE_NEHALEM;
-#endif			
        case 14:
 	  // Skylake
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 7:
 	    // Xeon Phi Knights Landing
-          if(support_avx())
-#ifndef NO_AVX2
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	case 12:
 	    // Apollo Lake
+	case 15:
+	    // Denverton		
 	    return CPUTYPE_NEHALEM;
 	}
 	break;
-      case 6:
+      case 6:  // family 6 exmodel 6
        switch (model) {
        case 6: // Cannon Lake
-#ifndef NO_AVX512
-	  return CPUTYPE_SKYLAKEX;
-#else
-	  if(support_avx())
-#ifndef NO_AVX2
-	  return CPUTYPE_HASWELL;
-#else
-	  return CPUTYPE_SANDYBRIDGE;
-#endif
+          if(support_avx512())
+            return CPUTYPE_SKYLAKEX;
+          if(support_avx2())
+            return CPUTYPE_HASWELL;
+          if(support_avx())
+	    return CPUTYPE_SANDYBRIDGE;
 	  else
 	  return CPUTYPE_NEHALEM;
-#endif			
        }
-      break;  
-      case 9:
-      case 8: 
+      break;
+      case 7: // family 6 exmodel 7
        switch (model) {
-	case 14: // Kaby Lake
-          if(support_avx())
-#ifndef NO_AVX2
+	case 10: // Goldmont Plus
+	    return CPUTYPE_NEHALEM;
+        case 14: // Ice Lake
+          if(support_avx512())
+            return CPUTYPE_SKYLAKEX;
+          if(support_avx2())
            return CPUTYPE_HASWELL;
-#else
+          if(support_avx())
+	    return CPUTYPE_SANDYBRIDGE;
+	  else
+	  return CPUTYPE_NEHALEM;
+        }
+      break;
+      case 9:
+      case 8:      
+        switch (model) {
+	case 14: // Kaby Lake and refreshes
+          if(support_avx2())
+            return CPUTYPE_HASWELL;
+          if(support_avx())
 	    return CPUTYPE_SANDYBRIDGE;
-#endif
          else
 	    return CPUTYPE_NEHALEM;
 	}
@@ -1393,7 +1431,11 @@ int get_cpuname(void){
    case 0x5:
      return CPUTYPE_AMDK6;
    case 0x6:
+#if defined(__x86_64__) || defined(__amd64__)
+      return CPUTYPE_BARCELONA;
+#else
      return CPUTYPE_ATHLON;
+#endif
    case 0xf:
      switch (exfamily) {
      case  0:
@@ -1469,6 +1511,26 @@ int get_cpuname(void){
    return CPUTYPE_AMD_UNKNOWN;
  }

+  if (vendor == VENDOR_HYGON){
+    switch (family) {
+    case 0xf:
+      switch (exfamily) {
+      case 9:
+          //Hygon Dhyana
+	  if(support_avx())
+#ifndef NO_AVX2
+	    return CPUTYPE_ZEN;
+#else
+	    return CPUTYPE_SANDYBRIDGE; // closer in architecture to Sandy Bridge than to Excavator
+#endif
+	  else
+	    return CPUTYPE_BARCELONA;
+        }
+      break;
+    }
+    return CPUTYPE_HYGON_UNKNOWN;
+  }
+
  if (vendor == VENDOR_CYRIX){
    switch (family) {
    case 0x4:
@@ -1590,7 +1652,8 @@ static char *cpuname[] = {
  "STEAMROLLER",
  "EXCAVATOR",
  "ZEN",
-  "SKYLAKEX"	
+  "SKYLAKEX",
+  "DHYANA"
 };

 static char *lowercpuname[] = {
@@ -1645,11 +1708,12 @@ static char *lowercpuname[] = {
  "steamroller",
  "excavator",
  "zen",
-  "skylakex"
+  "skylakex",
+  "dhyana"
 };

 static char *corename[] = {
-  "UNKOWN",
+  "UNKNOWN",
  "80486",
  "P5",
  "P6",
@@ -1677,7 +1741,8 @@ static char *corename[] = {
  "STEAMROLLER",
  "EXCAVATOR",
  "ZEN",
-  "SKYLAKEX"	
+  "SKYLAKEX",
+  "DHYANA"
 };

 static char *corename_lower[] = {
@@ -1709,7 +1774,8 @@ static char *corename_lower[] = {
  "steamroller",
  "excavator",
  "zen",
-  "skylakex"	
+  "skylakex",
+  "dhyana"
 };


@@ -1752,7 +1818,11 @@ int get_coretype(void){
 	case  4:
 	case  5:
 	case  6:
+#if defined(__x86_64__) || defined(__amd64__)
+	  return CORE_CORE2;
+#else
 	  return CORE_P6;
+#endif
 	case  7:
 	  return CORE_KATMAI;
 	case  8:
@@ -1936,6 +2006,38 @@ int get_coretype(void){
 	    return CORE_NEHALEM;
        }
 	break;
+      case 6:
+        if (model == 6)
+#ifndef NO_AVX512
+	    return CORE_SKYLAKEX;
+#else
+	  if(support_avx())
+#ifndef NO_AVX2
+	    return CORE_HASWELL;
+#else
+	    return CORE_SANDYBRIDGE;
+#endif
+	  else
+	    return CORE_NEHALEM;
+#endif			
+        break;    	
+      case 7:
+        if (model == 10) 
+            return CORE_NEHALEM;
+        if (model == 14)
+#ifndef NO_AVX512
+	    return CORE_SKYLAKEX;
+#else
+	  if(support_avx())
+#ifndef NO_AVX2
+	    return CORE_HASWELL;
+#else
+	    return CORE_SANDYBRIDGE;
+#endif
+	  else
+	    return CORE_NEHALEM;
+#endif			
+        break;    	
      case 9:
      case 8:
        if (model == 14) { // Kaby Lake 
@@ -1959,7 +2061,11 @@ int get_coretype(void){

  if (vendor == VENDOR_AMD){
    if (family <= 0x5) return CORE_80486;
+#if defined(__x86_64__) || defined(__amd64__)
+    if (family <= 0xe) return CORE_BARCELONA;
+#else
    if (family <= 0xe) return CORE_ATHLON;
+#endif
    if (family == 0xf){
      if ((exfamily == 0) || (exfamily == 2)) return CORE_OPTERON;
      else if (exfamily == 5) return CORE_BOBCAT;
@@ -2026,6 +2132,23 @@ int get_coretype(void){
    }
  }

+  if (vendor == VENDOR_HYGON){
+    if (family == 0xf){
+        if (exfamily == 9) {
+	  if(support_avx())
+#ifndef NO_AVX2
+	    return CORE_ZEN;
+#else
+	    return CORE_SANDYBRIDGE; // closer in architecture to Sandy Bridge than to Excavator
+#endif
+	  else
+	    return CORE_BARCELONA;
+	} else {
+		return CORE_BARCELONA;
+	}
+    }
+  }
+
  if (vendor == VENDOR_CENTAUR) {
    switch (family) {
    case 0x6:
@@ -2112,6 +2235,8 @@ void get_cpuconfig(void){
    if (features & HAVE_SSE4A)   printf("#define HAVE_SSE4A\n");
    if (features & HAVE_SSE5 )   printf("#define HAVE_SSSE5\n");
    if (features & HAVE_AVX )    printf("#define HAVE_AVX\n");
+    if (features & HAVE_AVX2 )    printf("#define HAVE_AVX2\n");
+    if (features & HAVE_AVX512VL )    printf("#define HAVE_AVX512VL\n");
    if (features & HAVE_3DNOWEX) printf("#define HAVE_3DNOWEX\n");
    if (features & HAVE_3DNOW)   printf("#define HAVE_3DNOW\n");
    if (features & HAVE_FMA4 )    printf("#define HAVE_FMA4\n");
@@ -2180,6 +2305,8 @@ void get_sse(void){
  if (features & HAVE_SSE4A)   printf("HAVE_SSE4A=1\n");
  if (features & HAVE_SSE5 )   printf("HAVE_SSSE5=1\n");
  if (features & HAVE_AVX )    printf("HAVE_AVX=1\n");
+  if (features & HAVE_AVX2 )    printf("HAVE_AVX2=1\n");
+  if (features & HAVE_AVX512VL )    printf("HAVE_AVX512VL=1\n");
  if (features & HAVE_3DNOWEX) printf("HAVE_3DNOWEX=1\n");
  if (features & HAVE_3DNOW)   printf("HAVE_3DNOW=1\n");
  if (features & HAVE_FMA4 )    printf("HAVE_FMA4=1\n");
@@ -27,20 +27,23 @@

 #include <string.h>

-#define CPU_GENERIC    	0
-#define CPU_Z13       	1
-#define CPU_Z14       	2
+#define CPU_GENERIC     0
+#define CPU_Z13         1
+#define CPU_Z14         2
+#define CPU_Z15         3

 static char *cpuname[] = {
  "ZARCH_GENERIC",
  "Z13",
-  "Z14"
+  "Z14",
+  "Z15"
 };

 static char *cpuname_lower[] = {
  "zarch_generic",
  "z13",
-  "z14"
+  "z14",
+  "z15"
 };

 int detect(void)
@@ -64,10 +67,10 @@ int detect(void)

  if (strstr(p, "2964")) return CPU_Z13;
  if (strstr(p, "2965")) return CPU_Z13;
-
-  /* detect z14, but fall back to z13 */
-  if (strstr(p, "3906")) return CPU_Z13;
-  if (strstr(p, "3907")) return CPU_Z13;
+  if (strstr(p, "3906")) return CPU_Z14;
+  if (strstr(p, "3907")) return CPU_Z14;
+  if (strstr(p, "8561")) return CPU_Z14;        // fallback z15 to z14
+  if (strstr(p, "8562")) return CPU_Z14;        // fallback z15 to z14

  return CPU_GENERIC;
 }
@@ -116,7 +119,14 @@ void get_cpuconfig(void)
 	  break;
 	case CPU_Z14:
 	  printf("#define Z14\n");
+	  printf("#define L1_DATA_SIZE 131072\n");
+	  printf("#define L1_DATA_LINESIZE 256\n");
+	  printf("#define L1_DATA_ASSOCIATIVE 8\n");
+	  printf("#define L2_SIZE 4194304\n");
+	  printf("#define L2_LINESIZE 256\n");
+	  printf("#define L2_ASSOCIATIVE 8\n");
 	  printf("#define DTB_DEFAULT_ENTRIES 64\n");
+	  printf("#define DTB_SIZE 4096\n");
 	  break;
 	}
 }
@@ -113,7 +113,7 @@ ARCH_X86
 ARCH_X86_64
 #endif

-#if defined(__powerpc___) || defined(__PPC__) || defined(_POWER)
+#if defined(__powerpc___) || defined(__PPC__) || defined(_POWER) || defined(__POWERPC__)
 ARCH_POWER
 #endif

@@ -6,6 +6,8 @@ TOPDIR = ..
 include $(TOPDIR)/Makefile.system

 override CFLAGS += -DADD$(BU) -DCBLAS
+override TARGET_ARCH=
+override TARGET_MACH=

 LIB = $(TOPDIR)/$(LIBNAME)

@@ -577,7 +577,7 @@
      SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC)
 *     ************************* STEST1 *****************************
 *
-*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN
+*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN
 *     REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE
 *     ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT.
 *
@@ -1503,6 +1503,8 @@ C     $      '      .' )
      NC = 0
      RESET = .TRUE.
      ERRMAX = RZERO
+      RALS = RONE
+      RBETS = RONE
 *
      DO 100 IN = 1, NIDIM
         N = IDIM( IN )
@@ -653,7 +653,7 @@
      SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC)
 *     ************************* STEST1 *****************************
 *
-*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN
+*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN
 *     REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE
 *     ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT.
 *
@@ -653,7 +653,7 @@
      SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC)
 *     ************************* STEST1 *****************************
 *
-*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN
+*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN
 *     REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE
 *     ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT.
 *
@@ -577,7 +577,7 @@
      SUBROUTINE STEST1(SCOMP1,STRUE1,SSIZE,SFAC)
 *     ************************* STEST1 *****************************
 *
-*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMODATE THE FORTRAN
+*     THIS IS AN INTERFACE SUBROUTINE TO ACCOMMODATE THE FORTRAN
 *     REQUIREMENT THAT WHEN A DUMMY ARGUMENT IS AN ARRAY, THE
 *     ACTUAL ARGUMENT MUST ALSO BE AN ARRAY OR AN ARRAY ELEMENT.
 *
@@ -1504,6 +1504,8 @@ C     $      '      .' )
      NC = 0
      RESET = .TRUE.
      ERRMAX = RZERO
+      RALS = RONE
+      RBETS = RONE
 *
      DO 100 IN = 1, NIDIM
         N = IDIM( IN )
@@ -5,7 +5,7 @@ T        LOGICAL FLAG, T TO STOP ON FAILURES.
 T        LOGICAL FLAG, T TO TEST ERROR EXITS.
 2        0 TO TEST COLUMN-MAJOR, 1 TO TEST ROW-MAJOR, 2 TO TEST BOTH
 16.0     THRESHOLD VALUE OF TEST RATIO
-6                 NUMBER OF VALUES OF N
+7                 NUMBER OF VALUES OF N
 1 2 3 5 7 9 35    VALUES OF N
 3                 NUMBER OF VALUES OF ALPHA
 0.0 1.0 0.7       VALUES OF ALPHA
@@ -5,7 +5,7 @@ T        LOGICAL FLAG, T TO STOP ON FAILURES.
 T        LOGICAL FLAG, T TO TEST ERROR EXITS.
 2        0 TO TEST COLUMN-MAJOR, 1 TO TEST ROW-MAJOR, 2 TO TEST BOTH
 16.0     THRESHOLD VALUE OF TEST RATIO
-6                 NUMBER OF VALUES OF N
+7                 NUMBER OF VALUES OF N
 0 1 2 3 5 9 35    VALUES OF N
 3                 NUMBER OF VALUES OF ALPHA
 0.0 1.0 0.7       VALUES OF ALPHA
@@ -54,16 +54,12 @@ int CNAME(BLASLONG m, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG incb, FLOAT *bu
    COPY_K(m, b, incb, buffer, 1);
  }

-/*FIXME the GEMV unrolling performed here was found to be broken, see issue 1332 */
-/* Multiplying DTB size by 100 is just a quick-and-dirty hack to disable it for now[B */
+  for (is = 0; is < m; is += DTB_ENTRIES){

-  for (is = 0; is < m; is += DTB_ENTRIES * 100){
-
-    min_i = MIN(m - is, DTB_ENTRIES * 100);
+    min_i = MIN(m - is, DTB_ENTRIES);

 #ifndef TRANSA
-    if (is > 0){
-fprintf(stderr,"WARNING unrolling of the trmv_U loop may give wrong results\n");    
+    if (is > 0){ 
      GEMV_N(is, min_i, 0, dp1,
 	     a + is * lda,  lda,
 	     B + is, 1,
@@ -346,7 +346,7 @@ int CNAME(BLASLONG m, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG incx, FLOAT *bu

    range_m[MAX_CPU_NUMBER - num_cpu - 1] = range_m[MAX_CPU_NUMBER - num_cpu] - width;
    range_n[num_cpu] = num_cpu * (((m + 15) & ~15) + 16);
-    if (range_n[num_cpu] > m) range_n[num_cpu] = m;
+    if (range_n[num_cpu] > m * num_cpu) range_n[num_cpu] = m * num_cpu;

    queue[num_cpu].mode    = mode;
    queue[num_cpu].routine = trmv_kernel;
@@ -386,7 +386,7 @@ int CNAME(BLASLONG m, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG incx, FLOAT *bu

    range_m[num_cpu + 1] = range_m[num_cpu] + width;
    range_n[num_cpu] = num_cpu * (((m + 15) & ~15) + 16);
-    if (range_n[num_cpu] > m) range_n[num_cpu] = m;
+    if (range_n[num_cpu] > m * num_cpu) range_n[num_cpu] = m * num_cpu;

    queue[num_cpu].mode    = mode;
    queue[num_cpu].routine = trmv_kernel;
@@ -338,7 +338,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -398,7 +398,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -463,7 +463,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -332,13 +332,16 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 #else
      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
-
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
        if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
        else
        	if (min_jj >= 2*GEMM_UNROLL_N) min_jj = 2*GEMM_UNROLL_N;
        	else
          		if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif


 	START_RPCC();
@@ -104,7 +104,7 @@ typedef struct {
 #define BETA_OPERATION(M_FROM, M_TO, N_FROM, N_TO, BETA, C, LDC) \
 	GEMM_BETA((M_TO) - (M_FROM), (N_TO - N_FROM), 0, \
 		  BETA[0], BETA[1], NULL, 0, NULL, 0, \
-		  (FLOAT *)(C) + (M_FROM) + (N_FROM) * (LDC) * COMPSIZE, LDC)
+		  (FLOAT *)(C) + ((M_FROM) + (N_FROM) * (LDC)) * COMPSIZE, LDC)
 #endif

 #ifndef ICOPYB_OPERATION
@@ -408,13 +408,13 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      /* Make sure if no one is using another buffer */
      for (i = 0; i < args -> nthreads; i++)
-	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;};
+	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;MB;};

      STOP_RPCC(waiting1);

      for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
 	min_jj = MIN(n_to, xxx + div_n) - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -441,7 +441,8 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      for (i = 0; i < args -> nthreads; i++)
 	job[mypos].working[i][CACHE_LINE_SIZE * bufferside] = (BLASLONG)buffer[bufferside];
-      }
+      WMB;
+	}

    current = mypos;

@@ -458,7 +459,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	  START_RPCC();

 	  /* thread has to wait */
-	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;};
+	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;MB;};

 	  STOP_RPCC(waiting2);

@@ -477,6 +478,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

 	if (m_to - m_from == min_i) {
 	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
+	WMB;
 	}
      }
    } while (current != mypos);
@@ -517,6 +519,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	if (is + min_i >= m_to) {
 	  /* Thread doesn't need this buffer any more */
 	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
+	WMB;
 	}
 	}

@@ -541,13 +544,13 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      /* Make sure if no one is using another buffer */
      for (i = 0; i < args -> nthreads; i++)
-	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;};
+	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;MB;};

      STOP_RPCC(waiting1);

      for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
 	min_jj = MIN(n_to, xxx + div_n) - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -595,7 +598,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	  START_RPCC();

 	  /* thread has to wait */
-	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;};
+	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;MB;};

 	  STOP_RPCC(waiting2);

@@ -613,6 +616,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

 	if (m_to - m_from == min_i) {
 	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
+	WMB;
 	}
      }
    } while (current != mypos);
@@ -677,13 +681,13 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

      /* Make sure if no one is using another buffer */
      for (i = 0; i < args -> nthreads; i++)
-	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;};
+	while (job[mypos].working[i][CACHE_LINE_SIZE * bufferside]) {YIELDING;MB;};

      STOP_RPCC(waiting1);

      for(jjs = xxx; jjs < MIN(n_to, xxx + div_n); jjs += min_jj){
 	min_jj = MIN(n_to, xxx + div_n) - jjs;
-	if (min_jj > GEMM3M_UNROLL_N) min_jj = GEMM3M_UNROLL_N;
+	if (min_jj > GEMM3M_UNROLL_N*3) min_jj = GEMM3M_UNROLL_N*3;

 	START_RPCC();

@@ -731,7 +735,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	  START_RPCC();

 	  /* thread has to wait */
-	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;};
+	  while(job[current].working[mypos][CACHE_LINE_SIZE * bufferside] == 0) {YIELDING;MB;};

 	  STOP_RPCC(waiting2);

@@ -748,8 +752,9 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	}

 	if (m_to - m_from == min_i) {
-	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
-	}
+	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] &= 0;
+	WMB;
+}
      }
    } while (current != mypos);

@@ -787,7 +792,8 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 #endif
 	if (is + min_i >= m_to) {
 	  /* Thread doesn't need this buffer any more */
-	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] = 0;
+	  job[current].working[mypos][CACHE_LINE_SIZE * bufferside] &= 0;
+	  WMB;
 	}
 	}

@@ -804,7 +810,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,

  for (i = 0; i < args -> nthreads; i++) {
    for (xxx = 0; xxx < DIVIDE_RATE; xxx++) {
-      while (job[mypos].working[i][CACHE_LINE_SIZE * xxx] ) {YIELDING;};
+      while (job[mypos].working[i][CACHE_LINE_SIZE * xxx] ) {YIELDING;MB;};
    }
  }

@@ -840,6 +846,15 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
 		       *range_n, FLOAT *sa, FLOAT *sb, BLASLONG mypos){

+#ifndef USE_OPENMP
+#ifndef OS_WINDOWS
+static pthread_mutex_t  level3_lock    = PTHREAD_MUTEX_INITIALIZER;
+#else
+CRITICAL_SECTION level3_lock;
+InitializeCriticalSection((PCRITICAL_SECTION)&level3_lock);
+#endif
+#endif
+
  blas_arg_t newarg;

  blas_queue_t queue[MAX_CPU_NUMBER];
@@ -869,6 +884,14 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
  mode  =  BLAS_SINGLE  | BLAS_REAL | BLAS_NODE;
 #endif

+#ifndef USE_OPENMP
+#ifndef OS_WINDOWS
+pthread_mutex_lock(&level3_lock);
+#else
+EnterCriticalSection((PCRITICAL_SECTION)&level3_lock);
+#endif
+#endif
+
  newarg.m        = args -> m;
  newarg.n        = args -> n;
  newarg.k        = args -> k;
@@ -973,6 +996,14 @@ static int gemm_driver(blas_arg_t *args, BLASLONG *range_m, BLASLONG
  free(job);
 #endif

+#ifndef USE_OPENMP
+#ifndef OS_WINDOWS
+  pthread_mutex_unlock(&level3_lock);
+#else
+  LeaveCriticalSection((PCRITICAL_SECTION)&level3_lock);
+#endif
+#endif
+
  return 0;
 }

@@ -365,12 +365,16 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
      /* Split local region of B into parts */
      for(jjs = js; jjs < MIN(n_to, js + div_n); jjs += min_jj){
 	min_jj = MIN(n_to, js + div_n) - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj >= 3*GEMM_UNROLL_N) min_jj = 3*GEMM_UNROLL_N;
 	else
          if (min_jj >= 2*GEMM_UNROLL_N) min_jj = 2*GEMM_UNROLL_N;
          else
            if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
        /* Copy part of local region of B into workspace */
 	START_RPCC();
 	OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs,
@@ -48,7 +48,7 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (

  BLASLONG width, i;
  BLASLONG n_from, n_to;
-  double dnum, nf, nt, di;
+  double dnum, nf, nt, di, dinum;

  int num_cpu;
  int mask = 0;
@@ -109,7 +109,11 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (
      if (nthreads - num_cpu > 1) {

 	di = (double)i;
-	width = (BLASLONG)(( sqrt(di * di + dnum) - di + mask)/(mask+1)) * (mask+1);
+	dinum = di * di +dnum;
+	if (dinum <0)
+	  width = (BLASLONG)(( - di + mask)/(mask+1)) * (mask+1);
+	else
+	  width = (BLASLONG)(( sqrt(dinum) - di + mask)/(mask+1)) * (mask+1);

 	if ((width <= 0) || (width > n_to - i)) width = n_to - i;

@@ -136,9 +140,7 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (

    nf = (double)(arg -> n - n_from);
    nt = (double)(arg -> n - n_to);
-
    dnum = (nt * nt - nf * nf) / (double)nthreads;
-
    num_cpu  = 0;

    range[0] = n_from;
@@ -149,8 +151,11 @@ int CNAME(int mode, blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n, int (
      if (nthreads - num_cpu > 1) {

 	di = (double)(arg -> n - i);
-	width = ((BLASLONG)((-sqrt(di * di + dnum) + di) + mask)/(mask+1)) * (mask+1);
-
+	dinum = di * di + dnum;
+	if (dinum<0)
+	  width = ((BLASLONG)(di + mask)/(mask+1)) * (mask+1);
+	else
+	  width = ((BLASLONG)((-sqrt(dinum) + di) + mask)/(mask+1)) * (mask+1);
 	if ((width <= 0) || (width > n_to - i)) width = n_to - i;

      } else {
@@ -135,10 +135,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

    for(jjs = js; jjs < js + min_j; jjs += min_jj){
      min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+      /* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+      if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
      if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
      else
      	if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
      START_RPCC();

      GEMM_ONCOPY(min_l, min_jj, b + (jjs * ldb) * COMPSIZE, ldb, sb + min_l * (jjs - js) * COMPSIZE);
@@ -201,10 +205,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
        if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
        else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 	START_RPCC();

 	GEMM_ONCOPY(min_l, min_jj, b + (ls + jjs * ldb) * COMPSIZE, ldb, sb + min_l * (jjs - js) * COMPSIZE);
@@ -292,10 +300,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

    for(jjs = js; jjs < js + min_j; jjs += min_jj){
      min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+      /* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+      if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
      if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
      else
        if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
      START_RPCC();

      GEMM_ONCOPY(min_l, min_jj, b + (m - min_l + jjs * ldb) * COMPSIZE, ldb,
@@ -358,10 +370,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
        if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
        else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 	START_RPCC();

 	GEMM_ONCOPY(min_l, min_jj, b + (ls - min_l + jjs * ldb) * COMPSIZE, ldb,
@@ -122,10 +122,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = 0; jjs < ls - js; jjs += min_jj){
 	min_jj = ls - js - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	GEMM_ONCOPY(min_l, min_jj, a + (ls + (js + jjs) * lda) * COMPSIZE, lda, sb + min_l * jjs * COMPSIZE);
 #else
@@ -142,10 +146,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = 0; jjs < min_l; jjs += min_jj){
 	min_jj = min_l - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	TRMM_OLNCOPY(min_l, min_jj, a, lda, ls, ls + jjs, sb + min_l * (ls - js + jjs) * COMPSIZE);
 #else
@@ -195,10 +203,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	GEMM_ONCOPY(min_l, min_jj, a + (ls + jjs * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);
 #else
@@ -246,10 +258,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = 0; jjs < min_l; jjs += min_jj){
 	min_jj = min_l - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	TRMM_OUNCOPY(min_l, min_jj, a, lda, ls, ls + jjs, sb + min_l * jjs * COMPSIZE);
 #else
@@ -267,10 +283,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = 0; jjs < js - ls - min_l; jjs += min_jj){
 	min_jj = js - ls - min_l - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	GEMM_ONCOPY(min_l, min_jj, a + (ls + (ls + min_l + jjs) * lda) * COMPSIZE, lda,
 		    sb + min_l * (min_l + jjs) * COMPSIZE);
@@ -324,10 +344,14 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *sa, FLO

      for(jjs = js; jjs < js + min_j; jjs += min_jj){
 	min_jj = min_j + js - jjs;
+#ifdef SKYLAKEX
+	/* the current AVX512 s/d/c/z GEMM kernel requires n>=6*GEMM_UNROLL_N to achieve the best performance */
+	if (min_jj >= 6*GEMM_UNROLL_N) min_jj = 6*GEMM_UNROLL_N;
+#else
 	if (min_jj > GEMM_UNROLL_N*3) min_jj = GEMM_UNROLL_N*3;
 	else
 	  if (min_jj > GEMM_UNROLL_N) min_jj = GEMM_UNROLL_N;
-
+#endif
 #ifndef TRANSA
 	GEMM_ONCOPY(min_l, min_jj, a + (ls + (jjs - min_j) * lda) * COMPSIZE, lda, sb + min_l * (jjs - js) * COMPSIZE);
 #else
@@ -47,7 +47,11 @@ GenerateNamedObjects("abs.c" "DOUBLE" "z_abs" 0 "" "" 1)
 GenerateNamedObjects("openblas_get_config.c;openblas_get_parallel.c" "" "" 0 "" "" 1)

 if (DYNAMIC_ARCH)
-  list(APPEND COMMON_SOURCES dynamic.c)
+  if (ARM64)
+    list(APPEND COMMON_SOURCES dynamic_arm64.c)
+  else ()  
+    list(APPEND COMMON_SOURCES dynamic.c)
+  endif ()  
 else ()
  list(APPEND COMMON_SOURCES parameter.c)
 endif ()
@@ -18,8 +18,16 @@ ifeq ($(DYNAMIC_ARCH), 1)
 ifeq ($(ARCH),arm64)
 COMMONOBJS	+=  dynamic_arm64.$(SUFFIX)
 else
+ifeq ($(ARCH),power)
+COMMONOBJS	+=  dynamic_power.$(SUFFIX)
+else
+ifeq ($(ARCH),zarch)
+COMMONOBJS += dynamic_zarch.$(SUFFIX)
+else
 COMMONOBJS	+=  dynamic.$(SUFFIX)
 endif
+endif
+endif
 else
 COMMONOBJS	+=  parameter.$(SUFFIX)
 endif
@@ -78,8 +86,16 @@ ifeq ($(DYNAMIC_ARCH), 1)
 ifeq ($(ARCH),arm64)
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_arm64.$(SUFFIX)
 else
+ifeq ($(ARCH),power)
+HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_power.$(SUFFIX)
+else
+ifeq ($(ARCH),zarch)
+HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic_zarch.$(SUFFIX)
+else
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) dynamic.$(SUFFIX)
 endif
+endif
+endif
 else
 HPLOBJS = memory.$(SUFFIX) xerbla.$(SUFFIX) parameter.$(SUFFIX)
 endif
@@ -109,7 +109,7 @@ extern unsigned int openblas_thread_timeout();
 /* equal to "OMP_NUM_THREADS - 1" and thread only wakes up when     */
 /* jobs is queued.                                                  */

-/* We need this grobal for cheking if initialization is finished.   */
+/* We need this global for checking if initialization is finished.  */
 int blas_server_avail   __attribute__((aligned(ATTRIBUTE_SIZE))) = 0;

 /* Local Variables */
@@ -150,8 +150,8 @@ static unsigned int thread_timeout = (1U << (THREAD_TIMEOUT));

 #ifdef MONITOR

-/* Monitor is a function to see thread's status for every seconds. */
-/* Usually it turns off and it's for debugging.                    */
+/* Monitor is a function to see thread's status for every second. */
+/* Usually it turns off and it's for debugging.                   */

 static pthread_t      monitor_thread;
 static int main_status[MAX_CPU_NUMBER];
@@ -50,7 +50,7 @@

 /* This is a thread implementation for Win32 lazy implementation */

-/* Thread server common infomation */
+/* Thread server common information */
 typedef struct{
  CRITICAL_SECTION lock;
  HANDLE filled;
@@ -61,7 +61,7 @@ typedef struct{

 } blas_pool_t;

-/* We need this global for cheking if initialization is finished.   */
+/* We need this global for checking if initialization is finished.   */
 int blas_server_avail = 0;

 /* Local Variables */
@@ -461,13 +461,22 @@ int BLASFUNC(blas_thread_shutdown)(void){
    SetEvent(pool.killed);

    for(i = 0; i < blas_num_threads - 1; i++){
-      WaitForSingleObject(blas_threads[i], 5);  //INFINITE);
+      // Could also just use WaitForMultipleObjects
+      DWORD wait_thread_value = WaitForSingleObject(blas_threads[i], 50);
+
 #ifndef OS_WINDOWSSTORE
-// TerminateThread is only available with WINAPI_DESKTOP and WINAPI_SYSTEM not WINAPI_APP in UWP
-      TerminateThread(blas_threads[i],0);
+      // TerminateThread is only available with WINAPI_DESKTOP and WINAPI_SYSTEM not WINAPI_APP in UWP
+      if (WAIT_OBJECT_0 != wait_thread_value) {
+        TerminateThread(blas_threads[i],0);
+      }
 #endif
+
+      CloseHandle(blas_threads[i]);
    }

+    CloseHandle(pool.filled);
+    CloseHandle(pool.killed);
+
    blas_server_avail = 0;
  }

@@ -274,6 +274,7 @@ extern gotoblas_t  gotoblas_SKYLAKEX;
 #define VENDOR_INTEL      1
 #define VENDOR_AMD        2
 #define VENDOR_CENTAUR    3
+#define VENDOR_HYGON	  4
 #define VENDOR_UNKNOWN   99

 #define BITMASK(a, b, c) ((((a) >> (b)) & (c)))
@@ -304,9 +305,49 @@ int support_avx(){
 #endif
 }

+int support_avx2(){
+#ifndef NO_AVX2
+  int eax, ebx, ecx=0, edx;
+  int ret=0;
+
+  if (!support_avx())
+    return 0;
+  cpuid(7, &eax, &ebx, &ecx, &edx);
+  if((ebx & (1<<7)) != 0)
+      ret=1;  //OS supports AVX2
+  return ret;
+#else
+  return 0;
+#endif
+}
+
+int support_avx512(){
+#if !defined(NO_AVX) && !defined(NO_AVX512)
+  int eax, ebx, ecx, edx;
+  int ret=0;
+
+  if (!support_avx())
+    return 0;
+  cpuid(7, &eax, &ebx, &ecx, &edx);
+  if((ebx & (1<<7)) == 0){
+      ret=0;  //OS does not even support AVX2
+  }
+  if((ebx & (1<<31)) != 0){
+    xgetbv(0, &eax, &edx);
+    if((eax & 0xe0) == 0xe0)
+      ret=1;  //OS supports AVX512VL
+  }
+  return ret;
+#else
+  return 0;
+#endif
+}
+
 extern void openblas_warning(int verbose, const char * msg);
 #define FALLBACK_VERBOSE 1
 #define NEHALEM_FALLBACK "OpenBLAS : Your OS does not support AVX instructions. OpenBLAS is using Nehalem kernels as a fallback, which may give poorer performance.\n"
+#define SANDYBRIDGE_FALLBACK "OpenBLAS : Your OS does not support AVX2 instructions. OpenBLAS is using Sandybridge kernels as a fallback, which may give poorer performance.\n"
+#define HASWELL_FALLBACK "OpenBLAS : Your OS does not support AVX512VL instructions. OpenBLAS is using Haswell kernels as a fallback, which may give poorer performance.\n"
 #define BARCELONA_FALLBACK "OpenBLAS : Your OS does not support AVX instructions. OpenBLAS is using Barcelona kernels as a fallback, which may give poorer performance.\n"

 static int get_vendor(void){
@@ -329,6 +370,7 @@ static int get_vendor(void){
  if (!strcmp(vendor.vchar, "GenuineIntel")) return VENDOR_INTEL;
  if (!strcmp(vendor.vchar, "AuthenticAMD")) return VENDOR_AMD;
  if (!strcmp(vendor.vchar, "CentaurHauls")) return VENDOR_CENTAUR;
+  if (!strcmp(vendor.vchar, "HygonGenuine")) return VENDOR_HYGON;

  if ((eax == 0) || ((eax & 0x500) != 0)) return VENDOR_INTEL;

@@ -403,18 +445,24 @@ static gotoblas_t *get_coretype(void){
 	}
 	//Intel Haswell
 	if (model == 12 || model == 15) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
 	//Intel Broadwell
 	if (model == 13) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
@@ -424,27 +472,36 @@ static gotoblas_t *get_coretype(void){
      case 4:
 		//Intel Haswell
 	if (model == 5 || model == 6) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
 	//Intel Broadwell
 	if (model == 7 || model == 15) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
 	//Intel Skylake
 	if (model == 14) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
@@ -457,72 +514,106 @@ static gotoblas_t *get_coretype(void){
      case 5:
 	//Intel Broadwell
 	if (model == 6) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
 	if (model == 5) {	
 	// Intel Skylake X
-#ifndef NO_AVX512
-	  return &gotoblas_SKYLAKEX;
-#else		
-	  if(support_avx())
+          if (support_avx512()) 
+	    return &gotoblas_SKYLAKEX;
+	  if(support_avx2()){
+	    openblas_warning(FALLBACK_VERBOSE, HASWELL_FALLBACK);
 	    return &gotoblas_HASWELL;
-	  else {
-	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
-	    return &gotoblas_NEHALEM;
-	  }
-#endif		
+          }
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
+          openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
+          return &gotoblas_NEHALEM;
+          }
 	}
 	//Intel Skylake
 	if (model == 14) {
-	  if(support_avx())
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
 	//Intel Phi Knights Landing
 	if (model == 7) {
-	  if(support_avx())
+	  if(support_avx2()){
+	    openblas_warning(FALLBACK_VERBOSE, HASWELL_FALLBACK);
 	    return &gotoblas_HASWELL;
-	  else{
+	  }  
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
-	//Apollo Lake
-	if (model == 12) { 
+	//Apollo Lake or Denverton
+	if (model == 12 || model == 15) { 
 	  return &gotoblas_NEHALEM;
 	}	
 	return NULL;
      case 6:
        if (model == 6) {
          // Cannon Lake
-#ifndef NO_AVX512
-	  return &gotoblas_SKYLAKEX;
-#else
-	  if(support_avx())
-#ifndef NO_AVX2
-	  return &gotoblas_HASWELL;
-#else
-	  return &gotoblas_SANDYBRIDGE;
-#endif
-	  else
-	  return &gotoblas_NEHALEM;
-#endif			
+	  if(support_avx2())
+	    return &gotoblas_HASWELL;
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
+	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
+	    return &gotoblas_NEHALEM;
+	  }
+        }
+        return NULL;  
+      case 7:
+	if (model == 10) // Goldmont Plus
+	   return &gotoblas_NEHALEM;
+        if (model == 14) {
+	// Ice Lake
+          if (support_avx512()) 
+	    return &gotoblas_SKYLAKEX;
+	  if(support_avx2()){
+	    openblas_warning(FALLBACK_VERBOSE, HASWELL_FALLBACK);
+	    return &gotoblas_HASWELL;
+          }
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
+          openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
+          return &gotoblas_NEHALEM;
+          }
        }
        return NULL;  
      case 9:
      case 8:
-	if (model == 14 ) { // Kaby Lake
-	  if(support_avx())
+	if (model == 14 ) { // Kaby Lake, Coffee Lake
+	  if(support_avx2())
 	    return &gotoblas_HASWELL;
-	  else{
+	  if(support_avx()) {
+	    openblas_warning(FALLBACK_VERBOSE, SANDYBRIDGE_FALLBACK);
+	    return &gotoblas_SANDYBRIDGE;
+	  } else {
 	    openblas_warning(FALLBACK_VERBOSE, NEHALEM_FALLBACK);
 	    return &gotoblas_NEHALEM; //OS doesn't support AVX. Use old kernels.
 	  }
@@ -535,7 +626,7 @@ static gotoblas_t *get_coretype(void){
    }
  }

-  if (vendor == VENDOR_AMD){
+  if (vendor == VENDOR_AMD || vendor == VENDOR_HYGON){
    if (family <= 0xe) {
        // Verify that CPU has 3dnow and 3dnowext before claiming it is Athlon
        cpuid(0x80000000, &eax, &ebx, &ecx, &edx);
@@ -615,6 +706,13 @@ static gotoblas_t *get_coretype(void){
 	    return &gotoblas_BARCELONA; //OS doesn't support AVX. Use old kernels.
 	  }
 	}
+      } else if (exfamily == 9) {
+	  if(support_avx())
+	    return &gotoblas_ZEN;
+	  else{
+	    openblas_warning(FALLBACK_VERBOSE, BARCELONA_FALLBACK);
+	    return &gotoblas_BARCELONA; //OS doesn't support AVX. Use old kernels.
+        }
      }else {
 	return &gotoblas_BARCELONA;
      }
@@ -37,17 +37,24 @@
 /*********************************************************************/

 #include "common.h"
+#if (defined OS_LINUX || defined OS_ANDROID)
 #include <asm/hwcap.h>
 #include <sys/auxv.h>
+#endif

 extern gotoblas_t  gotoblas_ARMV8;
+extern gotoblas_t  gotoblas_CORTEXA53;
 extern gotoblas_t  gotoblas_CORTEXA57;
+extern gotoblas_t  gotoblas_CORTEXA72;
+extern gotoblas_t  gotoblas_CORTEXA73;
+extern gotoblas_t  gotoblas_FALKOR;
 extern gotoblas_t  gotoblas_THUNDERX;
 extern gotoblas_t  gotoblas_THUNDERX2T99;
+extern gotoblas_t  gotoblas_TSV110;

 extern void openblas_warning(int verbose, const char * msg);

-#define NUM_CORETYPES    4
+#define NUM_CORETYPES    9

 /*
 * In case asm/hwcap.h is outdated on the build system, make sure
@@ -63,17 +70,27 @@ extern void openblas_warning(int verbose, const char * msg);

 static char *corename[] = {
  "armv8",
+  "cortexa53",
  "cortexa57",
+  "cortexa72",
+  "cortexa73",
+  "falkor",
  "thunderx",
  "thunderx2t99",
+  "tsv110",
  "unknown"
 };

 char *gotoblas_corename(void) {
  if (gotoblas == &gotoblas_ARMV8)        return corename[ 0];
-  if (gotoblas == &gotoblas_CORTEXA57)    return corename[ 1];
-  if (gotoblas == &gotoblas_THUNDERX)     return corename[ 2];
-  if (gotoblas == &gotoblas_THUNDERX2T99) return corename[ 3];
+  if (gotoblas == &gotoblas_CORTEXA53)    return corename[ 1];
+  if (gotoblas == &gotoblas_CORTEXA57)    return corename[ 2];
+  if (gotoblas == &gotoblas_CORTEXA72)    return corename[ 3];
+  if (gotoblas == &gotoblas_CORTEXA73)    return corename[ 4];
+  if (gotoblas == &gotoblas_FALKOR)       return corename[ 5];
+  if (gotoblas == &gotoblas_THUNDERX)     return corename[ 6];
+  if (gotoblas == &gotoblas_THUNDERX2T99) return corename[ 7];
+  if (gotoblas == &gotoblas_TSV110)       return corename[ 8];
  return corename[NUM_CORETYPES];
 }

@@ -94,9 +111,14 @@ static gotoblas_t *force_coretype(char *coretype) {
  switch (found)
  {
    case  0: return (&gotoblas_ARMV8);
-    case  1: return (&gotoblas_CORTEXA57);
-    case  2: return (&gotoblas_THUNDERX);
-    case  3: return (&gotoblas_THUNDERX2T99);
+    case  1: return (&gotoblas_CORTEXA53);
+    case  2: return (&gotoblas_CORTEXA57);
+    case  3: return (&gotoblas_CORTEXA72);
+    case  4: return (&gotoblas_CORTEXA73);
+    case  5: return (&gotoblas_FALKOR);
+    case  6: return (&gotoblas_THUNDERX);
+    case  7: return (&gotoblas_THUNDERX2T99);
+    case  8: return (&gotoblas_TSV110);
  }
  snprintf(message, 128, "Core not found: %s\n", coretype);
  openblas_warning(1, message);
@@ -105,13 +127,17 @@ static gotoblas_t *force_coretype(char *coretype) {

 static gotoblas_t *get_coretype(void) {
  int implementer, variant, part, arch, revision, midr_el1;
-  
+
+#if (defined OS_LINUX || defined OS_ANDROID)
  if (!(getauxval(AT_HWCAP) & HWCAP_CPUID)) {
    char coremsg[128];
    snprintf(coremsg, 128, "Kernel lacks cpuid feature support. Auto detection of core type failed !!!\n");
    openblas_warning(1, coremsg);
    return NULL;
  }
+#else
+   return NULL;
+#endif

  get_cpu_ftr(MIDR_EL1, midr_el1);
  /*
@@ -130,10 +156,14 @@ static gotoblas_t *get_coretype(void) {
    case 0x41: // ARM
      switch (part)
      {
-        case 0xd07: // Cortex A57
-        case 0xd08: // Cortex A72
        case 0xd03: // Cortex A53
+          return &gotoblas_CORTEXA53;
+        case 0xd07: // Cortex A57
          return &gotoblas_CORTEXA57;
+        case 0xd08: // Cortex A72
+          return &gotoblas_CORTEXA72;
+        case 0xd09: // Cortex A73
+          return &gotoblas_CORTEXA73;
      }
      break;
    case 0x42: // Broadcom
@@ -152,6 +182,20 @@ static gotoblas_t *get_coretype(void) {
          return &gotoblas_THUNDERX2T99;
      }
      break;
+    case 0x48: // HiSilicon
+      switch (part)
+      {
+        case 0xd01: // tsv110
+          return &gotoblas_TSV110;
+      }
+      break;
+    case 0x51: // Qualcomm
+      switch (part)
+      {
+        case 0xc00: // Falkor
+          return &gotoblas_FALKOR;
+      }
+      break;
  }
  return NULL;
 }
@@ -0,0 +1,110 @@
+
+#include "common.h"
+
+extern gotoblas_t gotoblas_POWER6;
+extern gotoblas_t gotoblas_POWER8;
+#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+extern gotoblas_t gotoblas_POWER9;
+#endif
+
+extern void openblas_warning(int verbose, const char *msg);
+
+static char *corename[] = {
+	"unknown",
+	"POWER6",
+	"POWER8",
+	"POWER9"
+};
+
+#define NUM_CORETYPES 4
+
+char *gotoblas_corename(void) {
+	if (gotoblas == &gotoblas_POWER6)	return corename[1];
+	if (gotoblas == &gotoblas_POWER8)	return corename[2];
+#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+	if (gotoblas == &gotoblas_POWER9)	return corename[3];
+#endif
+	return corename[0];
+}
+
+static gotoblas_t *get_coretype(void) {
+
+	if (__builtin_cpu_is("power6") || __builtin_cpu_is("power6x"))
+		return &gotoblas_POWER6;
+	if (__builtin_cpu_is("power8"))
+		return &gotoblas_POWER8;
+#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+	if (__builtin_cpu_is("power9"))
+		return &gotoblas_POWER9;
+#endif
+	return NULL;
+}
+
+static gotoblas_t *force_coretype(char * coretype) {
+
+	int i ;
+	int found = -1;
+	char message[128];
+
+	for ( i = 0 ; i < NUM_CORETYPES; i++)
+	{
+		if (!strncasecmp(coretype, corename[i], 20))
+		{
+			found = i;
+			break;
+		}
+	}
+
+	switch (found)
+	{
+	case  1: return (&gotoblas_POWER6);
+	case  2: return (&gotoblas_POWER8);
+#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+	case  3: return (&gotoblas_POWER9);
+#endif
+	default: return NULL;
+	}
+	snprintf(message, 128, "Core not found: %s\n", coretype);
+	openblas_warning(1, message);
+}
+
+void gotoblas_dynamic_init(void) {
+
+	char coremsg[128];
+	char coren[22];
+	char *p;
+
+
+	if (gotoblas) return;
+
+	p = getenv("OPENBLAS_CORETYPE");
+	if ( p )
+	{
+		gotoblas = force_coretype(p);
+	}
+	else
+	{
+		gotoblas = get_coretype();
+	}
+
+	if (gotoblas == NULL)
+	{
+		snprintf(coremsg, 128, "Falling back to POWER8 core\n");
+		openblas_warning(1, coremsg);
+		gotoblas = &gotoblas_POWER8;
+	}
+
+	if (gotoblas && gotoblas -> init) {
+		strncpy(coren,gotoblas_corename(),20);
+		sprintf(coremsg, "Core: %s\n",coren);
+		openblas_warning(2, coremsg);
+		gotoblas -> init();
+	} else {
+		openblas_warning(0, "OpenBLAS : Architecture Initialization failed. No initialization function found.\n");
+		exit(1);
+	}
+}
+
+void gotoblas_dynamic_quit(void) {
+	gotoblas = NULL;
+}
@@ -0,0 +1,131 @@
+
+#include "common.h"
+
+extern gotoblas_t gotoblas_Z13;
+extern gotoblas_t gotoblas_Z14;
+//extern gotoblas_t gotoblas_Z15;
+//#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+//extern gotoblas_t gotoblas_Z14;
+//#endif
+
+#define NUM_CORETYPES 4
+
+extern void openblas_warning(int verbose, const char* msg);
+
+static char* corename[] = {
+	"unknown",
+	"Z13",
+	"Z14",
+//	"Z15",
+	"ZARCH_GENERIC",
+};
+
+char* gotoblas_corename(void) {
+	if (gotoblas == &gotoblas_Z13)	return corename[1];
+	if (gotoblas == &gotoblas_Z14)	return corename[2];
+//	if (gotoblas == &gotoblas_Z15)	return corename[3];
+//#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+//	if (gotoblas == &gotoblas_POWER9)	return corename[3];
+//#endif
+	return corename[0]; // try generic?
+}
+
+// __builtin_cpu_is is not supported by zarch
+static gotoblas_t* get_coretype(void) {
+	FILE* infile;
+	char buffer[512], * p;
+
+	p = (char*)NULL;
+	infile = fopen("/proc/sysinfo", "r");
+	while (fgets(buffer, sizeof(buffer), infile)) {
+		if (!strncmp("Type", buffer, 4)) {
+			p = strchr(buffer, ':') + 2;
+#if 0
+			fprintf(stderr, "%s\n", p);
+#endif
+			break;
+		}
+	}
+
+	fclose(infile);
+
+	if (strstr(p, "2964")) return &gotoblas_Z13;
+	if (strstr(p, "2965")) return &gotoblas_Z13;
+	if (strstr(p, "3906")) return &gotoblas_Z14;
+	if (strstr(p, "3907")) return &gotoblas_Z14;
+	if (strstr(p, "8561")) return &gotoblas_Z14;        // fallback z15 to z14
+	if (strstr(p, "8562")) return &gotoblas_Z14;        // fallback z15 to z14
+
+	return NULL; // should be ZARCH_GENERIC
+}
+
+static gotoblas_t* force_coretype(char* coretype) {
+
+	int i;
+	int found = -1;
+	char message[128];
+
+	for (i = 0; i < NUM_CORETYPES; i++)
+	{
+		if (!strncasecmp(coretype, corename[i], 20))
+		{
+			found = i;
+			break;
+		}
+	}
+
+	switch (found)
+	{
+	case  1: return (&gotoblas_Z13);
+	case  2: return (&gotoblas_Z14);
+//	case  3: return (&gotoblas_Z15);
+//#if (!defined C_GCC) || (GCC_VERSION >= 60000)
+//	case  3: return (&gotoblas_POWER9);
+//#endif
+	default: return NULL;
+	}
+	snprintf(message, 128, "Core not found: %s\n", coretype);
+	openblas_warning(1, message);
+}
+
+void gotoblas_dynamic_init(void) {
+
+	char coremsg[128];
+	char coren[22];
+	char* p;
+
+
+	if (gotoblas) return;
+
+	p = getenv("OPENBLAS_CORETYPE");
+	if (p)
+	{
+		gotoblas = force_coretype(p);
+	}
+	else
+	{
+		gotoblas = get_coretype();
+	}
+
+	if (gotoblas == NULL)
+	{
+		snprintf(coremsg, 128, "Falling back to Z14 core\n");
+		openblas_warning(1, coremsg);
+		gotoblas = &gotoblas_Z14;
+	}
+
+	if (gotoblas && gotoblas->init) {
+		strncpy(coren, gotoblas_corename(), 20);
+		sprintf(coremsg, "Core: %s\n", coren);
+		openblas_warning(2, coremsg);
+		gotoblas->init();
+	}
+	else {
+		openblas_warning(0, "OpenBLAS : Architecture Initialization failed. No initialization function found.\n");
+		exit(1);
+	}
+}
+
+void gotoblas_dynamic_quit(void) {
+	gotoblas = NULL;
+}
@@ -765,7 +765,7 @@ int gotoblas_set_affinity(int pos) {

  int mynode = 1;

-  /* if number of threads is larger than inital condition */
+  /* if number of threads is larger than initial condition */
  if (pos < 0) {
      sched_setaffinity(0, sizeof(cpu_orig_mask), &cpu_orig_mask[0]);
      return 0;
@@ -857,7 +857,14 @@ void gotoblas_affinity_init(void) {
  common -> shmid = pshmid;

  if (common -> magic != SH_MAGIC) {
+
+#if defined(__GLIBC_PREREQ)
+#if __GLIBC_PREREQ(2, 7)
    cpu_set_t *cpusetp;
+#else
+    cpu_set_t cpuset;
+#endif
+#endif    
    int nums;
    int ret;

@@ -890,7 +897,7 @@ void gotoblas_affinity_init(void) {
    }
    CPU_FREE(cpusetp);
 #else
-    ret = sched_getaffinity(0,sizeof(cpu_set_t), cpusetp);
+    ret = sched_getaffinity(0,sizeof(cpu_set_t), &cpuset);
    if (ret!=0) {
        common->num_procs = nums;
    } else {
@@ -898,11 +905,11 @@ void gotoblas_affinity_init(void) {
    int i;
    int n = 0;
    for (i=0;i<nums;i++)
-        if (CPU_ISSET(i,cpusetp)) n++;
+        if (CPU_ISSET(i,&cpuset)) n++;
    common->num_procs = n;
    }
 #else
-    common->num_procs = CPU_COUNT(sizeof(cpu_set_t),cpusetp);
+    common->num_procs = CPU_COUNT(&cpuset);
    }
 #endif

@@ -129,7 +129,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <unistd.h>
 #endif

-#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN)
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN)
 #include <sys/sysctl.h>
 #include <sys/resource.h>
 #endif
@@ -192,51 +192,74 @@ void goto_set_num_threads(int num_threads) {};

 #else

-#if defined(OS_LINUX) || defined(OS_SUNOS) || defined(OS_NETBSD)
+#if defined(OS_LINUX) || defined(OS_SUNOS)
 #ifndef NO_AFFINITY
 int get_num_procs(void);
 #else
 int get_num_procs(void) {
  static int nums = 0;
-cpu_set_t *cpusetp;
-size_t size;
-int ret;
-int i,n;
+  cpu_set_t cpuset,*cpusetp;
+  size_t size;
+  int ret;
+
+#if defined(__GLIBC_PREREQ)
+#if !__GLIBC_PREREQ(2, 7)
+  int i;
+#if !__GLIBC_PREREQ(2, 6)
+  int n;
+#endif
+#endif
+#endif

  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
 #if !defined(OS_LINUX)
-     return nums;
+  return nums;
 #endif

 #if !defined(__GLIBC_PREREQ)
-   return nums;
+  return nums;
 #else
 #if !__GLIBC_PREREQ(2, 3)
-   return nums;
+  return nums;
 #endif

 #if !__GLIBC_PREREQ(2, 7)
-  ret = sched_getaffinity(0,sizeof(cpu_set_t), cpusetp);
+  ret = sched_getaffinity(0,sizeof(cpuset), &cpuset);
  if (ret!=0) return nums;
  n=0;
  #if !__GLIBC_PREREQ(2, 6)
  for (i=0;i<nums;i++)
-     if (CPU_ISSET(i,cpusetp)) n++;
+     if (CPU_ISSET(i,&cpuset)) n++;
  nums=n;
  #else
-  nums = CPU_COUNT(sizeof(cpu_set_t),cpusetp);
+  nums = CPU_COUNT(sizeof(cpuset),&cpuset);
  #endif
  return nums;
 #else
-  cpusetp = CPU_ALLOC(nums);
-  if (cpusetp == NULL) return nums;
-  size = CPU_ALLOC_SIZE(nums);
-  ret = sched_getaffinity(0,size,cpusetp);
-  if (ret!=0) return nums;
-  ret = CPU_COUNT_S(size,cpusetp);
-  if (ret > 0 && ret < nums) nums = ret;
-  CPU_FREE(cpusetp);
-  return nums;
+  if (nums >= CPU_SETSIZE) {
+    cpusetp = CPU_ALLOC(nums);
+      if (cpusetp == NULL) {
+        return nums;
+      }
+    size = CPU_ALLOC_SIZE(nums);
+    ret = sched_getaffinity(0,size,cpusetp);
+    if (ret!=0) {
+      CPU_FREE(cpusetp);
+      return nums;
+    }
+    ret = CPU_COUNT_S(size,cpusetp);
+    if (ret > 0 && ret < nums) nums = ret;	
+    CPU_FREE(cpusetp);
+    return nums;
+  } else {
+    ret = sched_getaffinity(0,sizeof(cpuset),&cpuset);
+    if (ret!=0) {
+      return nums;
+    }
+    ret = CPU_COUNT(&cpuset);
+    if (ret > 0 && ret < nums) nums = ret;	
+    return nums;
+  }
 #endif
 #endif
 }
@@ -289,7 +312,7 @@ int get_num_procs(void) {

 #endif

-#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY)
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY)

 int get_num_procs(void) {

@@ -381,7 +404,7 @@ extern int openblas_goto_num_threads_env();
 extern int openblas_omp_num_threads_env();

 int blas_get_cpu_number(void){
-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  int max_num;
 #endif
  int blas_goto_num   = 0;
@@ -389,7 +412,7 @@ int blas_get_cpu_number(void){

  if (blas_num_threads) return blas_num_threads;

-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  max_num = get_num_procs();
 #endif

@@ -413,7 +436,7 @@ int blas_get_cpu_number(void){
  else if (blas_omp_num > 0) blas_num_threads = blas_omp_num;
  else blas_num_threads = MAX_CPU_NUMBER;

-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  if (blas_num_threads > max_num) blas_num_threads = max_num;
 #endif

@@ -799,7 +822,7 @@ static void *alloc_qalloc(void *address){

 static void alloc_windows_free(struct alloc_t *alloc_info){

-  VirtualFree(alloc_info, allocation_block_size, MEM_DECOMMIT);
+  VirtualFree(alloc_info, 0, MEM_RELEASE);

 }

@@ -912,7 +935,7 @@ static void alloc_hugetlb_free(struct alloc_t *alloc_info){

 #ifdef OS_WINDOWS

-  VirtualFree(alloc_info, allocation_block_size, MEM_LARGE_PAGES | MEM_DECOMMIT);
+  VirtualFree(alloc_info, 0, MEM_LARGE_PAGES | MEM_RELEASE);

 #endif

@@ -1290,6 +1313,13 @@ void blas_memory_free_nolock(void * map_address) {
  free(map_address);
 }

+#ifdef SMP
+void blas_thread_memory_cleanup(void) {
+    blas_memory_cleanup((void*)get_memory_table());
+}
+#endif
+
+
 void blas_shutdown(void){
 #ifdef SMP
  BLASFUNC(blas_thread_shutdown)();
@@ -1299,7 +1329,7 @@ void blas_shutdown(void){
  /* Only cleanupIf we were built for threading and TLS was initialized */
  if (local_storage_key)
 #endif
-    blas_memory_cleanup((void*)get_memory_table());
+    blas_thread_memory_cleanup();

 #ifdef SEEK_ADDRESS
  base_address      = 0UL;
@@ -1486,6 +1516,14 @@ void DESTRUCTOR gotoblas_quit(void) {

  blas_shutdown();

+#if defined(SMP)
+#if defined(OS_WINDOWS)
+  TlsFree(local_storage_key);
+#else
+  pthread_key_delete(local_storage_key);
+#endif		
+#endif
+
 #ifdef PROFILE
   moncontrol (0);
 #endif
@@ -1521,7 +1559,7 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD  ul_reason_for_call, LPVOID lpReser
      break;
    case DLL_THREAD_DETACH:
 #if defined(SMP)
-      blas_memory_cleanup((void*)get_memory_table());
+      blas_thread_memory_cleanup();
 #endif
      break;
    case DLL_PROCESS_DETACH:
@@ -1584,6 +1622,7 @@ void gotoblas_dummy_for_PGI(void) {
  gotoblas_init();
  gotoblas_quit();

+#if __PGIC__ < 19
 #if 0
  asm ("\t.section\t.ctors,\"aw\",@progbits; .align 8; .quad gotoblas_init; .section .text");
  asm ("\t.section\t.dtors,\"aw\",@progbits; .align 8; .quad gotoblas_quit; .section .text");
@@ -1591,13 +1630,16 @@ void gotoblas_dummy_for_PGI(void) {
  asm (".section .init,\"ax\"; call gotoblas_init@PLT; .section .text");
  asm (".section .fini,\"ax\"; call gotoblas_quit@PLT; .section .text");
 #endif
+#endif
 }
 #endif

 #else
+/* USE_TLS / COMPILE_TLS not set */
+
 #include <errno.h>

-#ifdef OS_WINDOWS
+#if defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT)
 #define ALLOC_WINDOWS
 #ifndef MEM_LARGE_PAGES
 #define MEM_LARGE_PAGES  0x20000000
@@ -1611,7 +1653,7 @@ void gotoblas_dummy_for_PGI(void) {
 #include <stdio.h>
 #include <fcntl.h>

-#ifndef OS_WINDOWS
+#if !defined(OS_WINDOWS) || defined(OS_CYGWIN_NT)
 #include <sys/mman.h>
 #ifndef NO_SYSV_IPC
 #include <sys/shm.h>
@@ -1631,7 +1673,7 @@ void gotoblas_dummy_for_PGI(void) {
 #include <sys/resource.h>
 #endif

-#if defined(OS_FREEBSD) || defined(OS_DARWIN)
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN)
 #include <sys/sysctl.h>
 #include <sys/resource.h>
 #endif
@@ -1670,9 +1712,12 @@ void gotoblas_dummy_for_PGI(void) {
 #elif (defined(OS_DARWIN) || defined(OS_SUNOS)) && defined(C_GCC)
 #define CONSTRUCTOR	__attribute__ ((constructor))
 #define DESTRUCTOR	__attribute__ ((destructor))
-#else
+#elif __GNUC__ && INIT_PRIORITY && ((GCC_VERSION >= 40300) || (CLANG_VERSION >= 20900))
 #define CONSTRUCTOR	__attribute__ ((constructor(101)))
 #define DESTRUCTOR	__attribute__ ((destructor(101)))
+#else
+#define CONSTRUCTOR	__attribute__ ((constructor))
+#define DESTRUCTOR	__attribute__ ((destructor))
 #endif

 #ifdef DYNAMIC_ARCH
@@ -1691,50 +1736,75 @@ void goto_set_num_threads(int num_threads) {};

 #else

-#if defined(OS_LINUX) || defined(OS_SUNOS) || defined(OS_NETBSD)
+#if defined(OS_LINUX) || defined(OS_SUNOS)
 #ifndef NO_AFFINITY
 int get_num_procs(void);
 #else
 int get_num_procs(void) {
+
  static int nums = 0;
-cpu_set_t *cpusetp;
-size_t size;
-int ret;
-int i,n;
+  cpu_set_t cpuset,*cpusetp;
+  size_t size;
+  int ret;
+
+#if defined(__GLIBC_PREREQ)
+#if !__GLIBC_PREREQ(2, 7)
+  int i;
+#if !__GLIBC_PREREQ(2, 6)
+  int n;
+#endif
+#endif
+#endif

  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
 #if !defined(OS_LINUX)
-     return nums;
+  return nums;
 #endif

 #if !defined(__GLIBC_PREREQ)
-   return nums;
+  return nums;
 #else
 #if !__GLIBC_PREREQ(2, 3)
-   return nums;
+  return nums;
 #endif

 #if !__GLIBC_PREREQ(2, 7)
-  ret = sched_getaffinity(0,sizeof(cpu_set_t), cpusetp);
+  ret = sched_getaffinity(0,sizeof(cpuset), &cpuset);
  if (ret!=0) return nums;
  n=0;
  #if !__GLIBC_PREREQ(2, 6)
  for (i=0;i<nums;i++)
-     if (CPU_ISSET(i,cpusetp)) n++;
+     if (CPU_ISSET(i,&cpuset)) n++;
  nums=n;
  #else
-  nums = CPU_COUNT(sizeof(cpu_set_t),cpusetp);
+  nums = CPU_COUNT(sizeof(cpuset),&cpuset);
  #endif
  return nums;
 #else
-  cpusetp = CPU_ALLOC(nums);
-  if (cpusetp == NULL) return nums;
-  size = CPU_ALLOC_SIZE(nums);
-  ret = sched_getaffinity(0,size,cpusetp);
-  if (ret!=0) return nums;
-  nums = CPU_COUNT_S(size,cpusetp);
-  CPU_FREE(cpusetp);
-  return nums;
+  if (nums >= CPU_SETSIZE) {
+    cpusetp = CPU_ALLOC(nums);
+      if (cpusetp == NULL) {
+        return nums;
+      }
+    size = CPU_ALLOC_SIZE(nums);
+    ret = sched_getaffinity(0,size,cpusetp);
+    if (ret!=0) {
+      CPU_FREE(cpusetp);
+      return nums;
+    }
+    ret = CPU_COUNT_S(size,cpusetp);
+    if (ret > 0 && ret < nums) nums = ret;	
+    CPU_FREE(cpusetp);
+    return nums;
+  } else {
+    ret = sched_getaffinity(0,sizeof(cpuset),&cpuset);
+    if (ret!=0) {
+      return nums;
+    }
+    ret = CPU_COUNT(&cpuset);
+    if (ret > 0 && ret < nums) nums = ret;	
+    return nums;
+  }
 #endif
 #endif
 }
@@ -1748,7 +1818,7 @@ int get_num_procs(void) {
  return nums;
 }
 #endif
-	
+
 #ifdef OS_HAIKU
 int get_num_procs(void) {
  static int nums = 0;
@@ -1785,7 +1855,7 @@ int get_num_procs(void) {

 #endif

-#if defined(OS_FREEBSD)
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY)

 int get_num_procs(void) {

@@ -1862,7 +1932,7 @@ void openblas_fork_handler()
  //   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
  // In the mean time build with USE_OPENMP=0 or link against another
  // implementation of OpenMP.
-#if !(defined(OS_WINDOWS) || defined(OS_ANDROID)) && defined(SMP_SERVER)
+#if !((defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT)) || defined(OS_ANDROID)) && defined(SMP_SERVER)
  int err;
  err = pthread_atfork ((void (*)(void)) BLASFUNC(blas_thread_shutdown), NULL, NULL);
  if(err != 0)
@@ -1875,7 +1945,7 @@ extern int openblas_goto_num_threads_env();
 extern int openblas_omp_num_threads_env();

 int blas_get_cpu_number(void){
-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  int max_num;
 #endif
  int blas_goto_num   = 0;
@@ -1883,11 +1953,11 @@ int blas_get_cpu_number(void){

  if (blas_num_threads) return blas_num_threads;

-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  max_num = get_num_procs();
 #endif

-  blas_goto_num = 0;
+  // blas_goto_num = 0;
 #ifndef USE_OPENMP
  blas_goto_num=openblas_num_threads_env();
  if (blas_goto_num < 0) blas_goto_num = 0;
@@ -1899,7 +1969,7 @@ int blas_get_cpu_number(void){

 #endif

-  blas_omp_num = 0;
+  // blas_omp_num = 0;
  blas_omp_num=openblas_omp_num_threads_env();
  if (blas_omp_num < 0) blas_omp_num = 0;

@@ -1907,7 +1977,7 @@ int blas_get_cpu_number(void){
  else if (blas_omp_num > 0) blas_num_threads = blas_omp_num;
  else blas_num_threads = MAX_CPU_NUMBER;

-#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_DARWIN) || defined(OS_ANDROID)
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
  if (blas_num_threads > max_num) blas_num_threads = max_num;
 #endif

@@ -1971,8 +2041,12 @@ static BLASULONG  alloc_lock = 0UL;

 static void alloc_mmap_free(struct release_t *release){

+if (!release->address) return;
+
  if (munmap(release -> address, BUFFER_SIZE)) {
-    printf("OpenBLAS : munmap failed\n");
+      int errsv=errno;
+       perror("OpenBLAS : munmap failed:");
+       printf("error code=%d,\trelease->address=%lx\n",errsv,release->address);
  }
 }

@@ -1994,11 +2068,21 @@ static void *alloc_mmap(void *address){
  }

  if (map_address != (void *)-1) {
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
    LOCK_COMMAND(&alloc_lock);
+#endif    
    release_info[release_pos].address = map_address;
    release_info[release_pos].func    = alloc_mmap_free;
    release_pos ++;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
    UNLOCK_COMMAND(&alloc_lock);
+#endif    
+  } else {
+#ifdef DEBUG  
+        int errsv=errno;
+       perror("OpenBLAS : mmap failed:");
+       printf("error code=%d,\tmap_address=%lx\n",errsv,map_address);
+#endif
  }

 #ifdef OS_LINUX
@@ -2140,14 +2224,18 @@ static void *alloc_mmap(void *address){
 #if defined(OS_LINUX) && !defined(NO_WARMUP)
  }
 #endif
-  LOCK_COMMAND(&alloc_lock);

  if (map_address != (void *)-1) {
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+    LOCK_COMMAND(&alloc_lock);
+#endif
    release_info[release_pos].address = map_address;
    release_info[release_pos].func    = alloc_mmap_free;
    release_pos ++;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+    UNLOCK_COMMAND(&alloc_lock);
+#endif
  }
-  UNLOCK_COMMAND(&alloc_lock);

  return map_address;
 }
@@ -2222,7 +2310,7 @@ static void *alloc_qalloc(void *address){

 static void alloc_windows_free(struct release_t *release){

-  VirtualFree(release -> address, BUFFER_SIZE, MEM_DECOMMIT);
+  VirtualFree(release -> address, 0, MEM_RELEASE);

 }

@@ -2344,7 +2432,7 @@ static void alloc_hugetlb_free(struct release_t *release){

 #ifdef OS_WINDOWS

-  VirtualFree(release -> address, BUFFER_SIZE, MEM_LARGE_PAGES | MEM_DECOMMIT);
+  VirtualFree(release -> address, 0, MEM_LARGE_PAGES | MEM_RELEASE);

 #endif

@@ -2515,7 +2603,7 @@ void *blas_memory_alloc(int procpos){

  int position;
 #if defined(WHEREAMI) && !defined(USE_OPENMP)
-  int mypos;
+  int mypos = 0;
 #endif

  void *map_address;
@@ -2546,6 +2634,11 @@ void *blas_memory_alloc(int procpos){
    NULL,
  };
  void *(**func)(void *address);
+
+#if defined(USE_OPENMP)
+  if (!memory_initialized) {
+#endif
+
  LOCK_COMMAND(&alloc_lock);

  if (!memory_initialized) {
@@ -2581,6 +2674,9 @@ void *blas_memory_alloc(int procpos){

  }
  UNLOCK_COMMAND(&alloc_lock);
+#if defined(USE_OPENMP)
+  }
+#endif

 #ifdef DEBUG
  printf("Alloc Start ...\n");
@@ -2595,13 +2691,17 @@ void *blas_memory_alloc(int procpos){

  do {
    if (!memory[position].used && (memory[position].pos == mypos)) {
+#if defined(SMP) && !defined(USE_OPENMP)
      LOCK_COMMAND(&alloc_lock);
-//      blas_lock(&memory[position].lock);
-
+#else      
+      blas_lock(&memory[position].lock);
+#endif
      if (!memory[position].used) goto allocation;
-
+#if defined(SMP) && !defined(USE_OPENMP)
      UNLOCK_COMMAND(&alloc_lock);
-//      blas_unlock(&memory[position].lock);
+#else
+      blas_unlock(&memory[position].lock);
+#endif      
    }

    position ++;
@@ -2613,21 +2713,26 @@ void *blas_memory_alloc(int procpos){

  position = 0;

+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
  LOCK_COMMAND(&alloc_lock);
+#endif
  do {
-/*    if (!memory[position].used) { */
-/*      blas_lock(&memory[position].lock);*/
-
+#if defined(USE_OPENMP)	  
+    if (!memory[position].used) { 
+      blas_lock(&memory[position].lock);
+#endif
      if (!memory[position].used) goto allocation;
      
-/*      blas_unlock(&memory[position].lock);*/
-/*    } */
-
+#if defined(USE_OPENMP)
+      blas_unlock(&memory[position].lock);      
+    }
+#endif
    position ++;

  } while (position < NUM_BUFFERS);
-  UNLOCK_COMMAND(&alloc_lock);
-
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
+  UNLOCK_COMMAND(&alloc_lock);	
+#endif
  goto error;

  allocation :
@@ -2637,10 +2742,11 @@ void *blas_memory_alloc(int procpos){
 #endif

  memory[position].used = 1;
-
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
  UNLOCK_COMMAND(&alloc_lock);
-/*  blas_unlock(&memory[position].lock);*/
-
+#else
+  blas_unlock(&memory[position].lock);	
+#endif
  if (!memory[position].addr) {
    do {
 #ifdef DEBUG
@@ -2657,7 +2763,7 @@ void *blas_memory_alloc(int procpos){

 #ifdef ALLOC_DEVICEDRIVER
 	if ((*func ==  alloc_devicedirver) && (map_address == (void *)-1)) {
-	    fprintf(stderr, "OpenBLAS Warning ... Physically contigous allocation was failed.\n");
+	    fprintf(stderr, "OpenBLAS Warning ... Physically contiguous allocation was failed.\n");
 	}
 #endif

@@ -2685,9 +2791,13 @@ void *blas_memory_alloc(int procpos){

    } while ((BLASLONG)map_address == -1);

+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
    LOCK_COMMAND(&alloc_lock);
+#endif    
    memory[position].addr = map_address;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
    UNLOCK_COMMAND(&alloc_lock);
+#endif

 #ifdef DEBUG
    printf("  Mapping Succeeded. %p(%d)\n", (void *)memory[position].addr, position);
@@ -2741,8 +2851,9 @@ void blas_memory_free(void *free_area){
 #endif

  position = 0;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
  LOCK_COMMAND(&alloc_lock);
-
+#endif
  while ((position < NUM_BUFFERS) && (memory[position].addr != free_area))
    position++;

@@ -2756,7 +2867,9 @@ void blas_memory_free(void *free_area){
  WMB;

  memory[position].used = 0;
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
  UNLOCK_COMMAND(&alloc_lock);
+#endif

 #ifdef DEBUG
  printf("Unmap Succeeded.\n\n");
@@ -2771,8 +2884,9 @@ void blas_memory_free(void *free_area){
  for (position = 0; position < NUM_BUFFERS; position++)
    printf("%4ld  %p : %d\n", position, memory[position].addr, memory[position].used);
 #endif
+#if (defined(SMP) || defined(USE_LOCKING)) && !defined(USE_OPENMP)
  UNLOCK_COMMAND(&alloc_lock);
-
+#endif
  return;
 }

@@ -2822,7 +2936,7 @@ void blas_shutdown(void){

 #if defined(OS_LINUX) && !defined(NO_WARMUP)

-#ifdef SMP
+#if defined(SMP) || defined(USE_LOCKING)
 #if   defined(USE_PTHREAD_LOCK)
 static pthread_mutex_t    init_lock = PTHREAD_MUTEX_INITIALIZER;
 #elif defined(USE_PTHREAD_SPINLOCK)
@@ -2847,7 +2961,7 @@ static void _touch_memory(blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n,
    if (hot_alloc != 2) {
 #endif

-#ifdef SMP
+#if defined(SMP) || defined(USE_LOCKING)
  LOCK_COMMAND(&init_lock);
 #endif

@@ -2857,7 +2971,7 @@ static void _touch_memory(blas_arg_t *arg, BLASLONG *range_m, BLASLONG *range_n,
    size    -= PAGESIZE;
  }

-#ifdef SMP
+#if defined(SMP) || defined(USE_LOCKING)
  UNLOCK_COMMAND(&init_lock);
 #endif

@@ -3090,7 +3204,7 @@ void gotoblas_dummy_for_PGI(void) {

  gotoblas_init();
  gotoblas_quit();
-
+#if __PGIC__ < 19
 #if 0
  asm ("\t.section\t.ctors,\"aw\",@progbits; .align 8; .quad gotoblas_init; .section .text");
  asm ("\t.section\t.dtors,\"aw\",@progbits; .align 8; .quad gotoblas_quit; .section .text");
@@ -3098,6 +3212,7 @@ void gotoblas_dummy_for_PGI(void) {
  asm (".section .init,\"ax\"; call gotoblas_init@PLT; .section .text");
  asm (".section .fini,\"ax\"; call gotoblas_quit@PLT; .section .text");
 #endif
+#endif	
 }
 #endif

@@ -38,21 +38,29 @@

 #include <stdio.h>
 #include "common.h"
-
-#ifndef SMP
-#define blas_cpu_number 1
-#else
-
-int blas_cpu_number = 1;
-
-int blas_get_cpu_number(void){
-
-  return blas_cpu_number;
-}
+#ifdef OS_LINUX
+#include <sys/sysinfo.h>
+#include <sched.h>
+#include <errno.h>
+#include <linux/unistd.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/resource.h>
 #endif

+#ifdef OS_HAIKU
+#include <unistd.h>
+#endif
+
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN)
+#include <sys/sysctl.h>
+#include <sys/resource.h>
+#endif
+
+
 #define FIXED_PAGESIZE 4096

+
 void *sa = NULL;
 void *sb = NULL;
 static double static_buffer[BUFFER_SIZE/sizeof(double)];
@@ -60,7 +68,7 @@ static double static_buffer[BUFFER_SIZE/sizeof(double)];
 void *blas_memory_alloc(int numproc){

  if (sa == NULL){
-#if 1
+#if 0
    sa = (void *)qalloc(QFAST, BUFFER_SIZE);
 #else
    sa = (void *)malloc(BUFFER_SIZE);
@@ -75,3 +83,296 @@ void blas_memory_free(void *free_area){
  return;
 }

+
+
+extern void openblas_warning(int verbose, const char * msg);
+
+#ifndef SMP
+
+#define blas_cpu_number 1
+#define blas_num_threads 1
+
+/* Dummy Function */
+int  goto_get_num_procs  (void) { return 1;};
+void goto_set_num_threads(int num_threads) {};
+
+#else
+
+#if defined(OS_LINUX) || defined(OS_SUNOS)
+#ifndef NO_AFFINITY
+int get_num_procs(void);
+#else
+int get_num_procs(void) {
+
+  static int nums = 0;
+  cpu_set_t cpuset,*cpusetp;
+  size_t size;
+  int ret;
+
+#if defined(__GLIBC_PREREQ)
+#if !__GLIBC_PREREQ(2, 7)
+  int i;
+#if !__GLIBC_PREREQ(2, 6)
+  int n;
+#endif
+#endif
+#endif
+
+  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
+#if !defined(OS_LINUX)
+  return nums;
+#endif
+
+/*
+#if !defined(__GLIBC_PREREQ)
+  return nums;
+#else
+ #if !__GLIBC_PREREQ(2, 3)
+  return nums;
+ #endif
+
+ #if !__GLIBC_PREREQ(2, 7)
+  ret = sched_getaffinity(0,sizeof(cpuset), &cpuset);
+  if (ret!=0) return nums;
+  n=0;
+  #if !__GLIBC_PREREQ(2, 6)
+  for (i=0;i<nums;i++)
+     if (CPU_ISSET(i,&cpuset)) n++;
+  nums=n;
+  #else
+  nums = CPU_COUNT(sizeof(cpuset),&cpuset);
+  #endif
+  return nums;
+ #else
+  if (nums >= CPU_SETSIZE) {
+    cpusetp = CPU_ALLOC(nums);
+      if (cpusetp == NULL) {
+        return nums;
+      }
+    size = CPU_ALLOC_SIZE(nums);
+    ret = sched_getaffinity(0,size,cpusetp);
+    if (ret!=0) {
+      CPU_FREE(cpusetp);
+      return nums;
+    }
+    ret = CPU_COUNT_S(size,cpusetp);
+    if (ret > 0 && ret < nums) nums = ret;	
+    CPU_FREE(cpusetp);
+    return nums;
+  } else {
+    ret = sched_getaffinity(0,sizeof(cpuset),&cpuset);
+    if (ret!=0) {
+      return nums;
+    }
+    ret = CPU_COUNT(&cpuset);
+    if (ret > 0 && ret < nums) nums = ret;	
+    return nums;
+  }
+ #endif
+#endif
+*/
+   return 1;
+}
+#endif
+#endif
+
+#ifdef OS_ANDROID
+int get_num_procs(void) {
+  static int nums = 0;
+  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
+  return nums;
+}
+#endif
+
+#ifdef OS_HAIKU
+int get_num_procs(void) {
+  static int nums = 0;
+  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
+  return nums;
+}
+#endif
+
+#ifdef OS_AIX
+int get_num_procs(void) {
+  static int nums = 0;
+  if (!nums) nums = sysconf(_SC_NPROCESSORS_CONF);
+  return nums;
+}
+#endif
+
+#ifdef OS_WINDOWS
+
+int get_num_procs(void) {
+
+  static int nums = 0;
+
+  if (nums == 0) {
+
+    SYSTEM_INFO sysinfo;
+
+    GetSystemInfo(&sysinfo);
+
+    nums = sysinfo.dwNumberOfProcessors;
+  }
+
+  return nums;
+}
+
+#endif
+
+#if defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY)
+
+int get_num_procs(void) {
+
+  static int nums = 0;
+
+  int m[2];
+  size_t len;
+
+  if (nums == 0) {
+    m[0] = CTL_HW;
+    m[1] = HW_NCPU;
+    len = sizeof(int);
+    sysctl(m, 2, &nums, &len, NULL, 0);
+  }
+
+  return nums;
+}
+
+#endif
+
+#if defined(OS_DARWIN)
+int get_num_procs(void) {
+  static int nums = 0;
+  size_t len;
+  if (nums == 0){
+    len = sizeof(int);
+    sysctlbyname("hw.physicalcpu", &nums, &len, NULL, 0);
+  }
+  return nums;
+}
+/*
+void set_stack_limit(int limitMB){
+  int result=0;
+  struct rlimit rl;
+  rlim_t StackSize;
+
+  StackSize=limitMB*1024*1024;
+  result=getrlimit(RLIMIT_STACK, &rl);
+  if(result==0){
+    if(rl.rlim_cur < StackSize){
+      rl.rlim_cur=StackSize;
+      result=setrlimit(RLIMIT_STACK, &rl);
+      if(result !=0){
+        fprintf(stderr, "OpenBLAS: set stack limit error =%d\n", result);
+      }
+    }
+  }
+}
+*/
+#endif
+
+
+/*
+OpenBLAS uses the numbers of CPU cores in multithreading.
+It can be set by openblas_set_num_threads(int num_threads);
+*/
+int blas_cpu_number  = 0;
+/*
+The numbers of threads in the thread pool.
+This value is equal or large than blas_cpu_number. This means some threads are sleep.
+*/
+int blas_num_threads = 0;
+
+int  goto_get_num_procs  (void) {
+  return blas_cpu_number;
+}
+
+void openblas_fork_handler()
+{
+  // This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
+  // built with "make USE_OPENMP=0".
+  // Hanging can still happen when OpenBLAS is built against the libgomp
+  // implementation of OpenMP. The problem is tracked at:
+  //   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
+  // In the mean time build with USE_OPENMP=0 or link against another
+  // implementation of OpenMP.
+#if !((defined(OS_WINDOWS) && !defined(OS_CYGWIN_NT)) || defined(OS_ANDROID)) && defined(SMP_SERVER)
+  int err;
+  err = pthread_atfork ((void (*)(void)) BLASFUNC(blas_thread_shutdown), NULL, NULL);
+  if(err != 0)
+    openblas_warning(0, "OpenBLAS Warning ... cannot install fork handler. You may meet hang after fork.\n");
+#endif
+}
+
+extern int openblas_num_threads_env();
+extern int openblas_goto_num_threads_env();
+extern int openblas_omp_num_threads_env();
+
+int blas_get_cpu_number(void){
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+  int max_num;
+#endif
+  int blas_goto_num   = 0;
+  int blas_omp_num    = 0;
+
+  if (blas_num_threads) return blas_num_threads;
+
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+  max_num = get_num_procs();
+#endif
+
+  // blas_goto_num = 0;
+#ifndef USE_OPENMP
+  blas_goto_num=openblas_num_threads_env();
+  if (blas_goto_num < 0) blas_goto_num = 0;
+
+  if (blas_goto_num == 0) {
+    blas_goto_num=openblas_goto_num_threads_env();
+    if (blas_goto_num < 0) blas_goto_num = 0;
+  }
+
+#endif
+
+  // blas_omp_num = 0;
+  blas_omp_num=openblas_omp_num_threads_env();
+  if (blas_omp_num < 0) blas_omp_num = 0;
+
+  if (blas_goto_num > 0) blas_num_threads = blas_goto_num;
+  else if (blas_omp_num > 0) blas_num_threads = blas_omp_num;
+  else blas_num_threads = MAX_CPU_NUMBER;
+
+#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || defined(OS_DRAGONFLY) || defined(OS_DARWIN) || defined(OS_ANDROID)
+  if (blas_num_threads > max_num) blas_num_threads = max_num;
+#endif
+
+  if (blas_num_threads > MAX_CPU_NUMBER) blas_num_threads = MAX_CPU_NUMBER;
+
+#ifdef DEBUG
+  printf( "Adjusted number of threads : %3d\n", blas_num_threads);
+#endif
+
+  blas_cpu_number = blas_num_threads;
+
+  return blas_num_threads;
+}
+#endif
+
+
+int openblas_get_num_procs(void) {
+#ifndef SMP
+  return 1;
+#else
+  return get_num_procs();
+#endif
+}
+
+int openblas_get_num_threads(void) {
+#ifndef SMP
+  return 1;
+#else
+  // init blas_cpu_number if needed
+  blas_get_cpu_number();
+  return blas_cpu_number;
+#endif
+}
@@ -35,12 +35,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 #include <string.h>

-#if defined(_WIN32) && defined(_MSC_VER)
-#if _MSC_VER < 1900
-#define snprintf _snprintf
-#endif
-#endif
-
 static char* openblas_config_str=""
 "OpenBLAS "
 VERSION
@@ -84,10 +78,10 @@ char tmpstr[20];
 #ifdef DYNAMIC_ARCH
  strcat(tmp_config_str, gotoblas_corename());
 #endif
-if (openblas_get_parallel() == 0)
-  sprintf(tmpstr, " SINGLE_THREADED");
-else 
-  snprintf(tmpstr,19," MAX_THREADS=%d",MAX_CPU_NUMBER);
+  if (openblas_get_parallel() == 0)
+    sprintf(tmpstr, " SINGLE_THREADED");
+  else 
+    snprintf(tmpstr,19," MAX_THREADS=%d",MAX_CPU_NUMBER);
  strcat(tmp_config_str, tmpstr);
  return tmp_config_str;
 }
@@ -105,6 +105,10 @@ $(LIBPREFIX).def : gensymbol
 libgoto_hpl.def : gensymbol
 	perl ./gensymbol win2khpl $(ARCH) dummy $(EXPRECISION) $(NO_CBLAS) $(NO_LAPACK) $(NO_LAPACKE) $(NEED2UNDERSCORES) $(ONLY_CBLAS) "$(SYMBOLPREFIX)" "$(SYMBOLSUFFIX)" $(BUILD_LAPACK_DEPRECATED) > $(@F)

+ifeq ($(OSNAME), Darwin)
+INTERNALNAME = $(LIBPREFIX).$(MAJOR_VERSION).dylib
+endif
+
 ifeq (, $(SYMBOLPREFIX)$(SYMBOLSUFFIX))
 $(LIBDYNNAME) : ../$(LIBNAME) osx.def
 else
@@ -114,9 +118,9 @@ $(LIBDYNNAME) : ../$(LIBNAME).osx.renamed osx.def
 endif
 ifneq (,$(filter 1 2,$(NOFORTRAN)))
 #only build without Fortran
-	$(CC) $(CFLAGS) $(LDFLAGS) -all_load -headerpad_max_install_names -install_name "$(CURDIR)/../$(LIBDYNNAME)" -dynamiclib -o ../$(LIBDYNNAME) $< -Wl,-exported_symbols_list,osx.def  $(FEXTRALIB)
+	$(CC) $(CFLAGS) $(LDFLAGS) -all_load -headerpad_max_install_names -install_name "$(CURDIR)/../$(INTERNALNAME)" -dynamiclib -o ../$(LIBDYNNAME) $< -Wl,-exported_symbols_list,osx.def  $(FEXTRALIB)
 else
-	$(FC) $(FFLAGS) $(LDFLAGS) -all_load -headerpad_max_install_names -install_name "$(CURDIR)/../$(LIBDYNNAME)" -dynamiclib -o ../$(LIBDYNNAME) $< -Wl,-exported_symbols_list,osx.def  $(FEXTRALIB)
+	$(FC) $(FFLAGS) $(LDFLAGS) -all_load -headerpad_max_install_names -install_name "$(CURDIR)/../$(INTERNALNAME)" -dynamiclib -o ../$(LIBDYNNAME) $< -Wl,-exported_symbols_list,osx.def  $(FEXTRALIB)
 endif

 dllinit.$(SUFFIX) : dllinit.c
@@ -141,6 +145,14 @@ else
 	$(OBJCOPY) --redefine-syms objcopy.def ../$(LIBNAME) ../$(LIBNAME).renamed
 ../$(LIBSONAME) : ../$(LIBNAME).renamed linktest.c
 endif
+
+ifeq ($(F_COMPILER), INTEL)
+	$(FC) $(FFLAGS) $(LDFLAGS) -shared -o ../$(LIBSONAME) \
+	-Wl,--whole-archive $< -Wl,--no-whole-archive \
+	-Wl,-soname,$(INTERNALNAME) $(EXTRALIB)
+	$(CC) $(CFLAGS) $(LDFLAGS) -w -o linktest linktest.c ../$(LIBSONAME) $(FEXTRALIB) && echo OK.
+else
+
 ifneq ($(C_COMPILER), LSB)
 	$(CC) $(CFLAGS) $(LDFLAGS) -shared -o ../$(LIBSONAME) \
 	-Wl,--whole-archive $< -Wl,--no-whole-archive \
@@ -152,6 +164,7 @@ else
 	-Wl,--whole-archive $< -Wl,--no-whole-archive \
 	-Wl,-soname,$(INTERNALNAME) $(EXTRALIB)
 	$(FC) $(CFLAGS) $(LDFLAGS) -w -o linktest linktest.c ../$(LIBSONAME) $(FEXTRALIB) && echo OK.
+endif
 endif
 	rm -f linktest

--- a/Show More
+++ b/Show More