Merging r292349:

------------------------------------------------------------------------
r292349 | jlpeyton | 2017-01-18 01:02:21 -0600 (Wed, 18 Jan 2017) | 6 lines

Fix memory error in case of reinit using kmp_set_defaults() for lock code.

The lock tables were being reallocated if kmp_set_defaults() was called.
In the env_init code it says that the user should be able to switch between
different KMP_CONSISTENCY_CHECK values which is what this change enables.

------------------------------------------------------------------------


git-svn-id: https://llvm.org/svn/llvm-project/openmp/branches/release_40@292678 91177308-0d34-0410-b5e6-96231b3b80d8

CMakeLists.txt

@@ -1,2 +1,5 @@
 cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+
+set(OPENMP_LLVM_TOOLS_DIR "" CACHE PATH "Path to LLVM tools for testing")
+
 add_subdirectory(runtime)

CREDITS.txt

@@ -18,7 +18,7 @@ D: Contributor to testsuite from OpenUH
 N: Barbara Chapman
 D: Contributor to testsuite from OpenUH
 
-N: University of Houston 
+N: University of Houston
 W: http://web.cs.uh.edu/~openuh/download/
 D: OpenUH test suite
 
@@ -49,5 +49,9 @@ N: Alp Toker
 E: alp@nuanti.com
 D: Making build work for FreeBSD.
 
-N: Cheng Wang 
+N: Cheng Wang
 D: Contributor to testsuite from OpenUH
+
+N: Diego Caballero
+E: diego.l.caballero@gmail.com
+D: Fork performance improvements

LICENSE.txt

@@ -14,7 +14,7 @@ software contained in this directory tree is included below.
 University of Illinois/NCSA
 Open Source License
 
-Copyright (c) 1997-2015 Intel Corporation
+Copyright (c) 1997-2016 Intel Corporation
 
 All rights reserved.
 
@@ -51,7 +51,7 @@ SOFTWARE.
 
 ==============================================================================
 
-Copyright (c) 1997-2014 Intel Corporation
+Copyright (c) 1997-2016 Intel Corporation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -122,3 +122,53 @@ conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
 PARTICULAR PURPOSE.
 
 ==============================================================================
+
+ARM Limited
+
+Software Grant License Agreement ("Agreement")
+
+Except for the license granted herein to you, ARM Limited ("ARM") reserves all
+right, title, and interest in and to the Software (defined below).
+
+Definition
+
+"Software" means the code and documentation as well as any original work of
+authorship, including any modifications or additions to an existing work, that
+is intentionally submitted by ARM to llvm.org (http://llvm.org) ("LLVM") for
+inclusion in, or documentation of, any of the products owned or managed by LLVM
+(the "Work"). For the purposes of this definition, "submitted" means any form of
+electronic, verbal, or written communication sent to LLVM or its
+representatives, including but not limited to communication on electronic
+mailing lists, source code control systems, and issue tracking systems that are
+managed by, or on behalf of, LLVM for the purpose of discussing and improving
+the Work, but excluding communication that is conspicuously marked otherwise.
+
+1. Grant of Copyright License. Subject to the terms and conditions of this
+   Agreement, ARM hereby grants to you and to recipients of the Software
+   distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge,
+   royalty-free, irrevocable copyright license to reproduce, prepare derivative
+   works of, publicly display, publicly perform, sublicense, and distribute the
+   Software and such derivative works.
+
+2. Grant of Patent License. Subject to the terms and conditions of this
+   Agreement, ARM hereby grants you and to recipients of the Software
+   distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge,
+   royalty-free, irrevocable (except as stated in this section) patent license
+   to make, have made, use, offer to sell, sell, import, and otherwise transfer
+   the Work, where such license applies only to those patent claims licensable
+   by ARM that are necessarily infringed by ARM's Software alone or by
+   combination of the Software with the Work to which such Software was
+   submitted. If any entity institutes patent litigation against ARM or any
+   other entity (including a cross-claim or counterclaim in a lawsuit) alleging
+   that ARM's Software, or the Work to which ARM has contributed constitutes
+   direct or contributory patent infringement, then any patent licenses granted
+   to that entity under this Agreement for the Software or Work shall terminate
+   as of the date such litigation is filed.
+
+Unless required by applicable law or agreed to in writing, the software is
+provided on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+either express or implied, including, without limitation, any warranties or
+conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE.
+
+==============================================================================

runtime/Build_With_CMake.txt

@@ -14,7 +14,7 @@ How to Build the LLVM* OpenMP* Runtime Library using CMake
 ==========================================================
 
 ==== Version of CMake required: v2.8.0 or above ====
- 
+
 ============================================
 How to call cmake initially, then repeatedly
 ============================================
@@ -40,9 +40,9 @@ How to call cmake initially, then repeatedly
   for an empty build directory, but other Build options are used at any time.
 - The file CMakeCache.txt which is created after the first call to cmake
   is a configuration file which holds all the values for the Build options.
-  These configuration values can be changed using a text editor to modify 
+  These configuration values can be changed using a text editor to modify
   CMakeCache.txt as opposed to using definitions on the command line.
-- To have cmake create a particular type of build generator file simply 
+- To have cmake create a particular type of build generator file simply
   inlude the -G <Generator name> option:
   $ cmake -G "Unix Makefiles" ...
   You can see a list of generators cmake supports by executing cmake with
@@ -106,7 +106,7 @@ Specify the Fortran compiler
 This option is Windows* Only
 
 -DLIBOMP_ARCH=i386|x86_64|arm|ppc64|ppc64le|aarch64|mic
-The default for the option is chosen based on the probing the compiler for 
+The default for the option is chosen based on the probing the compiler for
 architecture macros (e.g., is __x86_64__ predefined by compiler?).
 
 ==== First values listed are the default value ====
@@ -116,19 +116,19 @@ Library type can be normal, profile, or stubs.
 -DCMAKE_BUILD_TYPE=Release|Debug|RelWithDebInfo
 Build type can be Release, Debug, or RelWithDebInfo.
 
--DLIBOMP_OMP_VERSION=41|40|30
-OpenMP version can be either 41, 40 or 30.
+-DLIBOMP_OMP_VERSION=45|40|30
+OpenMP version can be either 45, 40 or 30.
 
 -DLIBOMP_MIC_ARCH=knc|knf
 This value is ignored if LIBOMP_ARCH != mic
 Intel(R) MIC Architecture, can be knf or knc.
-  
+
 -DLIBOMP_FORTRAN_MODULES=off|on
 Should the Fortran modules be created (requires Fortran compiler)
 
--DLIBOMP_USE_ADAPTIVE_LOCKS=on|off       
-Should adaptive (TSX-based) locks be included?  
-These are x86 specific.  This feature is turned on by default 
+-DLIBOMP_USE_ADAPTIVE_LOCKS=on|off
+Should adaptive (TSX-based) locks be included?
+These are x86 specific.  This feature is turned on by default
 for i386 and x86_64.  Otherwise, it is turned off.
 
 -DLIBOMP_USE_INTERNODE_ALIGNMENT=off|on
@@ -143,6 +143,11 @@ symbols are supported (Linux, some BSD* variants).  It is off
 by default for Windows and Mac, but on for other Unix based operating
 systems.
 
+-DLIBOMP_ENABLE_SHARED=on|off
+Shared library instead of static library? (Note: static libraries are not
+supported on Windows). If LIBOMP_ENABLE_SHARED is off, then static OpenMP
+libraries will be built instead of dynamic ones.
+
 -DLIBOMP_OMPT_SUPPORT=off|on
 Should OMPT support be included in the build? (Not supported on Windows)
 If LIBOMP_OMPT_SUPPORT is off, then both ompt_blame and ompt_trace are ignored.
@@ -168,13 +173,21 @@ http://www.open-mpi.org/projects/hwloc
 Default: /usr/local
 This option is only used if LIBOMP_USE_HWLOC is on.
 Specifies install location of Hwloc. The configuration system will look for
-hwloc.h in ${LIBOMP_HWLOC_INSTALL_DIR}/include and the library in 
+hwloc.h in ${LIBOMP_HWLOC_INSTALL_DIR}/include and the library in
 ${LIBOMP_HWLOC_INSTALL_DIR}/lib.
 
+-DLIBOMP_LLVM_LIT_EXECUTABLE=/path/to/llvm-lit
+Default: search in PATH
+Specifiy full path to llvm-lit executable for running tests.
+
+-DOPENMP_LLVM_TOOLS_DIR=/path/to/built/llvm/tools
+Default: search for tools in path
+Additional path to search for LLVM tools needed by tests.
+
 ================================
 How to append flags to the build
 ================================
-- These flags are *appended*.  They do not 
+- These flags are *appended*.  They do not
   overwrite any of the preset flags.
 -DLIBOMP_CPPFLAGS=<space-separated flags> -- Additional C preprocessor flags
 -DLIBOMP_CFLAGS=<space-separated flags>   -- Additional C compiler flags

runtime/CMakeLists.txt

@@ -17,7 +17,8 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH})
 
 # Standalone build or part of LLVM?
 set(LIBOMP_STANDALONE_BUILD FALSE)
-if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}" OR
+if(OPENMP_STANDALONE_BUILD OR
+   "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}" OR
    "${CMAKE_SOURCE_DIR}/runtime" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   project(libomp C CXX)
   set(LIBOMP_STANDALONE_BUILD TRUE)
@@ -38,7 +39,7 @@ if(${LIBOMP_STANDALONE_BUILD})
   # If adding a new architecture, take a look at cmake/LibompGetArchitecture.cmake
   libomp_get_architecture(LIBOMP_DETECTED_ARCH)
   set(LIBOMP_ARCH ${LIBOMP_DETECTED_ARCH} CACHE STRING
-    "The architecture to build for (x86_64/i386/arm/ppc64/ppc64le/aarch64/mic).")
+    "The architecture to build for (x86_64/i386/arm/ppc64/ppc64le/aarch64/mic/mips/mips64).")
   # Allow user to choose a suffix for the installation directory.
   set(LIBOMP_LIBDIR_SUFFIX "" CACHE STRING
     "suffix of lib installation directory e.g., 64 => lib64")
@@ -65,6 +66,8 @@ else() # Part of LLVM build
     set(LIBOMP_ARCH x86_64)
   elseif(LIBOMP_NATIVE_ARCH STREQUAL "x86_64")
     set(LIBOMP_ARCH x86_64)
+  elseif(LIBOMP_NATIVE_ARCH MATCHES "powerpc64le")
+    set(LIBOMP_ARCH ppc64le)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "powerpc")
     set(LIBOMP_ARCH ppc64)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "aarch64")
@@ -81,16 +84,22 @@ else() # Part of LLVM build
   set(LIBOMP_ENABLE_ASSERTIONS ${LLVM_ENABLE_ASSERTIONS})
   set(LIBOMP_ENABLE_WERROR ${LLVM_ENABLE_WERROR})
 endif()
-libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc64 ppc64le aarch64 mic)
+libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc64 ppc64le aarch64 mic mips mips64)
 
 set(LIBOMP_LIB_TYPE normal CACHE STRING
   "Performance,Profiling,Stubs library (normal/profile/stubs)")
 libomp_check_variable(LIBOMP_LIB_TYPE normal profile stubs)
-set(LIBOMP_OMP_VERSION 41 CACHE STRING
-  "The OpenMP version (41/40/30)")
-libomp_check_variable(LIBOMP_OMP_VERSION 41 40 30)
+set(LIBOMP_OMP_VERSION 45 CACHE STRING
+  "The OpenMP version (45/40/30)")
+if(LIBOMP_OMP_VERSION EQUAL 41)
+  libomp_warning_say("LIBOMP_OMP_VERSION=41 is deprecated and will be removed in a later version. Please use 45.")
+  set(LIBOMP_OMP_VERSION 45)
+endif()
+libomp_check_variable(LIBOMP_OMP_VERSION 45 40 30)
 # Set the OpenMP Year and Month assiociated with version
-if(${LIBOMP_OMP_VERSION} GREATER 40 OR ${LIBOMP_OMP_VERSION} EQUAL 40)
+if(${LIBOMP_OMP_VERSION} GREATER 45 OR ${LIBOMP_OMP_VERSION} EQUAL 45)
+  set(LIBOMP_OMP_YEAR_MONTH 201511)
+elseif(${LIBOMP_OMP_VERSION} GREATER 40 OR ${LIBOMP_OMP_VERSION} EQUAL 40)
   set(LIBOMP_OMP_YEAR_MONTH 201307)
 elseif(${LIBOMP_OMP_VERSION} GREATER 30 OR ${LIBOMP_OMP_VERSION} EQUAL 30)
   set(LIBOMP_OMP_YEAR_MONTH 201107)
@@ -112,6 +121,14 @@ set(LIBOMP_OSX_ARCHITECTURES "${CMAKE_OSX_ARCHITECTURES}" CACHE STRING
   "For Mac builds, semicolon separated list of architectures to build for universal fat binary.")
 set(CMAKE_OSX_ARCHITECTURES ${LIBOMP_OSX_ARCHITECTURES})
 
+# Should @rpath be used for dynamic libraries on Mac?
+# The if(NOT DEFINED) is there to guard a cached value of the variable if one
+# exists so there is no interference with what the user wants.  Also, no cache entry
+# is created so there are no inadvertant effects on other parts of LLVM.
+if(NOT DEFINED CMAKE_MACOSX_RPATH)
+  set(CMAKE_MACOSX_RPATH TRUE)
+endif()
+
 # User specified flags.  These are appended to the configured flags.
 set(LIBOMP_CFLAGS "" CACHE STRING
   "Appended user specified C compiler flags.")
@@ -141,7 +158,7 @@ set(LIBOMP_USE_HWLOC FALSE CACHE BOOL
 set(LIBOMP_HWLOC_INSTALL_DIR /usr/local CACHE PATH
   "Install path for hwloc library")
 
-# Get the build number from kmp_version.c
+# Get the build number from kmp_version.cpp
 libomp_get_build_number("${CMAKE_CURRENT_SOURCE_DIR}" LIBOMP_VERSION_BUILD)
 math(EXPR LIBOMP_VERSION_BUILD_YEAR "${LIBOMP_VERSION_BUILD}/10000")
 math(EXPR LIBOMP_VERSION_BUILD_MONTH_DAY "${LIBOMP_VERSION_BUILD}%10000")
@@ -158,6 +175,8 @@ set(PPC64BE FALSE)
 set(PPC64LE FALSE)
 set(PPC64 FALSE)
 set(MIC FALSE)
+set(MIPS64 FALSE)
+set(MIPS FALSE)
 if("${LIBOMP_ARCH}" STREQUAL "i386" OR "${LIBOMP_ARCH}" STREQUAL "32")    # IA-32 architecture
   set(IA32 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "x86_64" OR "${LIBOMP_ARCH}" STREQUAL "32e") # Intel(R) 64 architecture
@@ -174,6 +193,10 @@ elseif("${LIBOMP_ARCH}" STREQUAL "aarch64") # AARCH64 architecture
   set(AARCH64 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "mic") # Intel(R) Many Integrated Core Architecture
   set(MIC TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "mips") # MIPS architecture
+    set(MIPS TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "mips64") # MIPS64 architecture
+    set(MIPS64 TRUE)
 endif()
 
 # Set some flags based on build_type
@@ -192,8 +215,9 @@ elseif("${libomp_build_type_lowercase}" STREQUAL "minsizerel")
   set(MINSIZEREL_BUILD TRUE)
 endif()
 
-# Include itt notify interface? Right now, always.
-set(LIBOMP_USE_ITT_NOTIFY TRUE)
+# Include itt notify interface?
+set(LIBOMP_USE_ITT_NOTIFY TRUE CACHE BOOL
+  "Enable ITT notify?")
 
 # normal, profile, stubs library.
 set(NORMAL_LIBRARY FALSE)
@@ -277,6 +301,14 @@ if(LIBOMP_STATS)
   set(LIBOMP_USE_STDCPPLIB TRUE)
 endif()
 
+# Shared library can be switched to a static library
+set(LIBOMP_ENABLE_SHARED TRUE CACHE BOOL
+  "Shared library instead of static library?")
+
+if(WIN32 AND NOT LIBOMP_ENABLE_SHARED)
+  libomp_error_say("Static libraries requested but not available on Windows")
+endif()
+
 # OMPT-support
 set(LIBOMP_OMPT_DEBUG FALSE CACHE BOOL
   "Trace OMPT initialization?")
@@ -290,6 +322,13 @@ if(LIBOMP_OMPT_SUPPORT AND (NOT LIBOMP_HAVE_OMPT_SUPPORT))
   libomp_error_say("OpenMP Tools Interface requested but not available")
 endif()
 
+# TSAN-support
+set(LIBOMP_TSAN_SUPPORT FALSE CACHE BOOL
+  "TSAN-support?")
+if(LIBOMP_TSAN_SUPPORT AND (NOT LIBOMP_HAVE_TSAN_SUPPORT))
+  libomp_error_say("TSAN functionality requested but not available")
+endif()
+
 # Error check hwloc support after config-ix has run
 if(LIBOMP_USE_HWLOC AND (NOT LIBOMP_HAVE_HWLOC))
   libomp_error_say("Hwloc requested but not available")
@@ -304,7 +343,22 @@ if(${STUBS_LIBRARY})
   set(LIBOMP_DEFAULT_LIB_NAME ${LIBOMP_DEFAULT_LIB_NAME}stubs)
 endif()
 set(LIBOMP_LIB_NAME ${LIBOMP_DEFAULT_LIB_NAME} CACHE STRING "Base OMP library name")
-set(LIBOMP_LIB_FILE ${LIBOMP_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+if(${LIBOMP_ENABLE_SHARED})
+  set(LIBOMP_LIBRARY_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX})
+  set(LIBOMP_LIBRARY_KIND SHARED)
+  set(LIBOMP_INSTALL_KIND LIBRARY)
+else()
+  set(LIBOMP_LIBRARY_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(LIBOMP_LIBRARY_KIND STATIC)
+  set(LIBOMP_INSTALL_KIND ARCHIVE)
+endif()
+
+set(LIBOMP_LIB_FILE ${LIBOMP_LIB_NAME}${LIBOMP_LIBRARY_SUFFIX})
+
+# Optional backwards compatibility aliases.
+set(LIBOMP_INSTALL_ALIASES TRUE CACHE BOOL
+  "Install libgomp and libiomp5 library aliases for backwards compatibility")
 
 # Print configuration after all variables are set.
 if(${LIBOMP_STANDALONE_BUILD})
@@ -315,7 +369,8 @@ if(${LIBOMP_STANDALONE_BUILD})
   endif()
   libomp_say("Build Type           -- ${CMAKE_BUILD_TYPE}")
   libomp_say("OpenMP Version       -- ${LIBOMP_OMP_VERSION}")
-  libomp_say("Lib Type             -- ${LIBOMP_LIB_TYPE}")
+  libomp_say("Library Kind         -- ${LIBOMP_LIBRARY_KIND}")
+  libomp_say("Library Type         -- ${LIBOMP_LIB_TYPE}")
   libomp_say("Fortran Modules      -- ${LIBOMP_FORTRAN_MODULES}")
   # will say development if all zeros
   if(${LIBOMP_VERSION_BUILD} STREQUAL 00000000)
@@ -326,6 +381,7 @@ if(${LIBOMP_STANDALONE_BUILD})
   libomp_say("Build                -- ${LIBOMP_BUILD}")
   libomp_say("Use Stats-gathering  -- ${LIBOMP_STATS}")
   libomp_say("Use Debugger-support -- ${LIBOMP_USE_DEBUGGER}")
+  libomp_say("Use ITT notify       -- ${LIBOMP_USE_ITT_NOTIFY}")
   libomp_say("Use OMPT-support     -- ${LIBOMP_OMPT_SUPPORT}")
   if(${LIBOMP_OMPT_SUPPORT})
     libomp_say("Use OMPT-blame       -- ${LIBOMP_OMPT_BLAME}")
@@ -333,9 +389,9 @@ if(${LIBOMP_STANDALONE_BUILD})
   endif()
   libomp_say("Use Adaptive locks   -- ${LIBOMP_USE_ADAPTIVE_LOCKS}")
   libomp_say("Use quad precision   -- ${LIBOMP_USE_QUAD_PRECISION}")
+  libomp_say("Use TSAN-support     -- ${LIBOMP_TSAN_SUPPORT}")
   libomp_say("Use Hwloc library    -- ${LIBOMP_USE_HWLOC}")
 endif()
 
 add_subdirectory(src)
 add_subdirectory(test)
-

runtime/README.txt

@@ -6,10 +6,10 @@ How to Build Documentation
 ==========================
 
 The main documentation is in Doxygen* format, and this distribution
-should come with pre-built PDF documentation in doc/Reference.pdf.  
+should come with pre-built PDF documentation in doc/Reference.pdf.
 However, an HTML version can be built by executing:
 
-% doxygen doc/doxygen/config 
+% doxygen doc/doxygen/config
 
 in the runtime directory.
 
@@ -45,18 +45,19 @@ For details about building, please look at Build_With_CMake.txt
 
 Architectures Supported
 =======================
-* IA-32 architecture 
+* IA-32 architecture
 * Intel(R) 64 architecture
 * Intel(R) Many Integrated Core Architecture
 * ARM* architecture
 * Aarch64 (64-bit ARM) architecture
 * IBM(R) Power architecture (big endian)
 * IBM(R) Power architecture (little endian)
+* MIPS and MIPS64 architecture
 
 Supported RTL Build Configurations
 ==================================
 
-Supported Architectures: IA-32 architecture, Intel(R) 64, and 
+Supported Architectures: IA-32 architecture, Intel(R) 64, and
 Intel(R) Many Integrated Core Architecture
 
               ----------------------------------------------
@@ -68,15 +69,15 @@ Intel(R) Many Integrated Core Architecture
 | Windows* OS |   Yes(1,4)    |  No         | No           |
 ------------------------------------------------------------
 
-(1) On IA-32 architecture and Intel(R) 64, icc/icl versions 12.x are 
+(1) On IA-32 architecture and Intel(R) 64, icc/icl versions 12.x are
     supported (12.1 is recommended).
 (2) GCC* version 4.7 is supported.
 (3) For icc on OS X*, OS X* version 10.5.8 is supported.
 (4) Intel(R) Many Integrated Core Architecture not supported.
-(5) On Intel(R) Many Integrated Core Architecture, icc/icl versions 13.0 
+(5) On Intel(R) Many Integrated Core Architecture, icc/icl versions 13.0
     or later are required.
 (6) Clang* version 3.3 is supported.
-(7) Clang* currently does not offer a software-implemented 128 bit extended 
+(7) Clang* currently does not offer a software-implemented 128 bit extended
     precision type.  Thus, all entry points reliant on this type are removed
     from the library and cannot be called in the user program.  The following
     functions are not available:

runtime/cmake/LibompExports.cmake

@@ -21,7 +21,7 @@ set(libomp_suffix)
 libomp_append(libomp_suffix .deb DEBUG_BUILD)
 libomp_append(libomp_suffix .dia RELWITHDEBINFO_BUILD)
 libomp_append(libomp_suffix .min MINSIZEREL_BUILD)
-if(NOT "${LIBOMP_OMP_VERSION}" STREQUAL "41")
+if(NOT "${LIBOMP_OMP_VERSION}" STREQUAL "45")
   libomp_append(libomp_suffix .${LIBOMP_OMP_VERSION})
 endif()
 libomp_append(libomp_suffix .s1 LIBOMP_STATS)
@@ -64,10 +64,12 @@ endif()
 if(${LIBOMP_FORTRAN_MODULES})
   add_custom_command(TARGET libomp-mod POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E make_directory ${LIBOMP_EXPORTS_MOD_DIR}
-    COMMAND ${CMAKE_COMMAND} -E copy omp_lib.h ${LIBOMP_EXPORTS_CMN_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy omp_lib.mod ${LIBOMP_EXPORTS_MOD_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy omp_lib_kinds.mod ${LIBOMP_EXPORTS_MOD_DIR}
   )
+  add_custom_command(TARGET omp POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy omp_lib.h ${LIBOMP_EXPORTS_CMN_DIR}
+  )
 endif()
 
 # Copy OpenMP library into exports/ directory post build

runtime/cmake/LibompGetArchitecture.cmake

@@ -42,6 +42,10 @@ function(libomp_get_architecture return_arch)
       #error ARCHITECTURE=ppc64le
     #elif defined(__powerpc64__)
       #error ARCHITECTURE=ppc64
+    #elif defined(__mips__) && defined(__mips64)
+      #error ARCHITECTURE=mips64
+    #elif defined(__mips__) && !defined(__mips64)
+      #error ARCHITECTURE=mips
     #else
       #error ARCHITECTURE=UnknownArchitecture
     #endif

runtime/cmake/LibompHandleFlags.cmake

@@ -28,6 +28,7 @@ function(libomp_get_c_and_cxxflags_common flags)
   set(flags_local)
   libomp_append(flags_local -std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG)
   libomp_append(flags_local -fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG)
+  libomp_append(flags_local -fno-rtti LIBOMP_HAVE_FNO_RTTI_FLAG)
   if(${LIBOMP_ENABLE_WERROR})
     libomp_append(flags_local -Werror LIBOMP_HAVE_WERROR_FLAG)
   endif()
@@ -191,8 +192,8 @@ function(libomp_get_gdflags gdflags)
   libomp_append(gdflags_local "-D stub" STUBS_LIBRARY)
   libomp_append(gdflags_local "-D HAVE_QUAD" LIBOMP_USE_QUAD_PRECISION)
   libomp_append(gdflags_local "-D USE_DEBUGGER" LIBOMP_USE_DEBUGGER)
-  if(${LIBOMP_OMP_VERSION} GREATER 41 OR ${LIBOMP_OMP_VERSION} EQUAL 41)
-    libomp_append(gdflags_local "-D OMP_41")
+  if(${LIBOMP_OMP_VERSION} GREATER 45 OR ${LIBOMP_OMP_VERSION} EQUAL 45)
+    libomp_append(gdflags_local "-D OMP_45")
   endif()
   if(${LIBOMP_OMP_VERSION} GREATER 40 OR ${LIBOMP_OMP_VERSION} EQUAL 40)
     libomp_append(gdflags_local "-D OMP_40")

runtime/cmake/LibompMicroTests.cmake

@@ -150,8 +150,8 @@ add_custom_target(libomp-test-execstack DEPENDS test-execstack/.success)
 add_custom_command(
   OUTPUT  test-execstack/.success
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-execstack
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
+  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.pl
+    --arch=${LIBOMP_PERL_SCRIPT_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
   COMMAND ${CMAKE_COMMAND} -E touch test-execstack/.success
   DEPENDS omp
 )
@@ -162,7 +162,7 @@ add_custom_command(
   OUTPUT  test-instr/.success
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-instr
   COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_ARCH} --show --mic-arch=${LIBOMP_MIC_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
+    --arch=${LIBOMP_PERL_SCRIPT_ARCH} --show --mic-arch=${LIBOMP_MIC_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
   COMMAND ${CMAKE_COMMAND} -E touch test-instr/.success
   DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl
 )
@@ -206,6 +206,9 @@ else()
     elseif(${PPC64})
       libomp_append(libomp_expected_library_deps libc.so.6)
       libomp_append(libomp_expected_library_deps ld64.so.1)
+    elseif(${MIPS} OR ${MIPS64})
+      libomp_append(libomp_expected_library_deps libc.so.6)
+      libomp_append(libomp_expected_library_deps ld.so.1)
     endif()
     libomp_append(libomp_expected_library_deps libpthread.so.0 IF_FALSE STUBS_LIBRARY)
     libomp_append(libomp_expected_library_deps libhwloc.so.5 LIBOMP_USE_HWLOC)
@@ -219,7 +222,7 @@ add_custom_command(
   OUTPUT  test-deps/.success
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-deps
   COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-depends.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_ARCH} --expected="${libomp_expected_library_deps}" ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
+    --arch=${LIBOMP_PERL_SCRIPT_ARCH} --expected="${libomp_expected_library_deps}" ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
   COMMAND ${CMAKE_COMMAND} -E touch test-deps/.success
   DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-depends.pl
 )

runtime/cmake/LibompUtils.cmake

@@ -102,6 +102,10 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "PPC64LE" PARENT_SCOPE)
   elseif(${AARCH64})
     set(${return_arch_string} "AARCH64" PARENT_SCOPE)
+  elseif(${MIPS})
+    set(${return_arch_string} "MIPS" PARENT_SCOPE)
+  elseif(${MIPS64})
+    set(${return_arch_string} "MIPS64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)
     libomp_warning_say("libomp_get_legal_arch(): Warning: Unknown architecture: Using ${LIBOMP_ARCH}")
@@ -129,12 +133,12 @@ function(libomp_check_variable var)
 endfunction()
 
 # void libomp_get_build_number(string src_dir, string* return_build_number);
-# - grab the eight digit build number (or 00000000) from kmp_version.c
+# - grab the eight digit build number (or 00000000) from kmp_version.cpp
 function(libomp_get_build_number src_dir return_build_number)
-  # sets file_lines_list to a list of all lines in kmp_version.c
-  file(STRINGS "${src_dir}/src/kmp_version.c" file_lines_list)
+  # sets file_lines_list to a list of all lines in kmp_version.cpp
+  file(STRINGS "${src_dir}/src/kmp_version.cpp" file_lines_list)
 
-  # runs through each line in kmp_version.c
+  # runs through each line in kmp_version.cpp
   foreach(line IN LISTS file_lines_list)
     # if the line begins with "#define KMP_VERSION_BUILD" then we take not of the build number
     string(REGEX MATCH "^[ \t]*#define[ \t]+KMP_VERSION_BUILD" valid "${line}")

runtime/cmake/config-ix.cmake

@@ -49,6 +49,7 @@ endfunction()
 # Checking C, CXX, Linker Flags
 check_cxx_compiler_flag(-std=c++11 LIBOMP_HAVE_STD_CPP11_FLAG)
 check_cxx_compiler_flag(-fno-exceptions LIBOMP_HAVE_FNO_EXCEPTIONS_FLAG)
+check_cxx_compiler_flag(-fno-rtti LIBOMP_HAVE_FNO_RTTI_FLAG)
 check_c_compiler_flag("-x c++" LIBOMP_HAVE_X_CPP_FLAG)
 check_c_compiler_flag(-Werror LIBOMP_HAVE_WERROR_FLAG)
 check_c_compiler_flag(-Wunused-function LIBOMP_HAVE_WNO_UNUSED_FUNCTION_FLAG)
@@ -144,8 +145,8 @@ endif()
 # Find perl executable
 # Perl is used to create omp.h (and other headers) along with kmp_i18n_id.inc and kmp_i18n_default.inc
 find_package(Perl REQUIRED)
-# The perl scripts take the --os= flag which expects a certain format for operating systems.  Until the
-# perl scripts are removed, the most portable way to handle this is to have all operating systems that
+# The perl scripts take the --os=/--arch= flags which expect a certain format for operating systems and arch's.
+# Until the perl scripts are removed, the most portable way to handle this is to have all operating systems that
 # are neither Windows nor Mac (Most Unix flavors) be considered lin to the perl scripts.  This is rooted
 # in that all the Perl scripts check the operating system and will fail if it isn't "valid".  This
 # temporary solution lets us avoid trying to enumerate all the possible OS values inside the Perl modules.
@@ -156,6 +157,15 @@ elseif(APPLE)
 else()
   set(LIBOMP_PERL_SCRIPT_OS lin)
 endif()
+if(IA32)
+  set(LIBOMP_PERL_SCRIPT_ARCH 32)
+elseif(MIC)
+  set(LIBOMP_PERL_SCRIPT_ARCH mic)
+elseif(INTEL64)
+  set(LIBOMP_PERL_SCRIPT_ARCH 32e)
+else()
+  set(LIBOMP_PERL_SCRIPT_ARCH ${LIBOMP_ARCH})
+endif()
 
 # Checking features
 # Check if version symbol assembler directives are supported
@@ -185,15 +195,15 @@ endif()
 if(${LIBOMP_STATS})
   check_c_source_compiles(
      "__thread int x;
-     int main(int argc, char** argv) 
+     int main(int argc, char** argv)
      { x = argc; return x; }"
      LIBOMP_HAVE___THREAD)
   check_c_source_compiles(
-     "int main(int argc, char** argv) 
+     "int main(int argc, char** argv)
      { unsigned long long t = __builtin_readcyclecounter(); return 0; }"
      LIBOMP_HAVE___BUILTIN_READCYCLECOUNTER)
   if(NOT LIBOMP_HAVE___BUILTIN_READCYCLECOUNTER)
-    if(${IA32} OR ${INTEL64})
+    if(${IA32} OR ${INTEL64} OR ${MIC})
       check_include_file(x86intrin.h LIBOMP_HAVE_X86INTRIN_H)
       libomp_append(CMAKE_REQUIRED_DEFINITIONS -DLIBOMP_HAVE_X86INTRIN_H LIBOMP_HAVE_X86INTRIN_H)
       check_c_source_compiles(
@@ -237,23 +247,28 @@ endif()
 
 # Check if HWLOC support is available
 if(${LIBOMP_USE_HWLOC})
-  if(WIN32)
-    set(LIBOMP_HAVE_HWLOC FALSE)
-    libomp_say("Using hwloc not supported on Windows yet")
-  else()
-    set(CMAKE_REQUIRED_INCLUDES ${LIBOMP_HWLOC_INSTALL_DIR}/include)
-    check_include_file(hwloc.h LIBOMP_HAVE_HWLOC_H)
-    set(CMAKE_REQUIRED_INCLUDES)
-    check_library_exists(hwloc hwloc_topology_init 
+  set(CMAKE_REQUIRED_INCLUDES ${LIBOMP_HWLOC_INSTALL_DIR}/include)
+  check_include_file(hwloc.h LIBOMP_HAVE_HWLOC_H)
+  set(CMAKE_REQUIRED_INCLUDES)
+  find_library(LIBOMP_HWLOC_LIBRARY
+    NAMES hwloc libhwloc
+    HINTS ${LIBOMP_HWLOC_INSTALL_DIR}/lib)
+  if(LIBOMP_HWLOC_LIBRARY)
+    check_library_exists(${LIBOMP_HWLOC_LIBRARY} hwloc_topology_init
       ${LIBOMP_HWLOC_INSTALL_DIR}/lib LIBOMP_HAVE_LIBHWLOC)
-    find_library(LIBOMP_HWLOC_LIBRARY hwloc ${LIBOMP_HWLOC_INSTALL_DIR}/lib)
     get_filename_component(LIBOMP_HWLOC_LIBRARY_DIR ${LIBOMP_HWLOC_LIBRARY} PATH)
-    if(LIBOMP_HAVE_HWLOC_H AND LIBOMP_HAVE_LIBHWLOC AND LIBOMP_HWLOC_LIBRARY)
-      set(LIBOMP_HAVE_HWLOC TRUE)
-    else()
-      set(LIBOMP_HAVE_HWLOC FALSE)
-      libomp_say("Could not find hwloc")
-    endif()
+  endif()
+  if(LIBOMP_HAVE_HWLOC_H AND LIBOMP_HAVE_LIBHWLOC AND LIBOMP_HWLOC_LIBRARY)
+    set(LIBOMP_HAVE_HWLOC TRUE)
+  else()
+    set(LIBOMP_HAVE_HWLOC FALSE)
+    libomp_say("Could not find hwloc")
   endif()
 endif()
 
+# Check if ThreadSanitizer support is available
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux" AND ${INTEL64})
+  set(LIBOMP_HAVE_TSAN_SUPPORT TRUE)
+else()
+  set(LIBOMP_HAVE_TSAN_SUPPORT FALSE)
+endif()

runtime/doc/doxygen/config

@@ -669,10 +669,10 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories
 # with spaces.
 
-INPUT                  = src  doc/doxygen/libomp_interface.h 
+INPUT                  = src  doc/doxygen/libomp_interface.h
 # The ittnotify code also has doxygen documentation, but if we include it here
 # it takes over from us!
-# src/thirdparty/ittnotify                         
+# src/thirdparty/ittnotify
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is
@@ -690,7 +690,7 @@ INPUT_ENCODING         = UTF-8
 # *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py
 # *.f90 *.f *.for *.vhd *.vhdl
 
-FILE_PATTERNS          = *.c *.h *.cpp 
+FILE_PATTERNS          = *.c *.h *.cpp
 # We may also want to include the asm files with appropriate ifdef to ensure
 # doxygen doesn't see the content, just the documentation...
 

runtime/doc/doxygen/header.tex

@@ -66,7 +66,7 @@ Intel, Xeon, and Intel Xeon Phi are trademarks of Intel Corporation in the U.S.
 
 This document is Copyright \textcopyright~\the\year the LLVM Project. It is
 subject to the same license terms as the LLVM OpenMP runtime.
- 
+
 \textsuperscript{*} Other names and brands may be claimed as the property of others.
 
 \clearemptydoublepage

runtime/src/CMakeLists.txt

@@ -20,17 +20,22 @@ endif()
 add_custom_command(
   OUTPUT  kmp_i18n_id.inc
   COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_ARCH} --prefix=kmp_i18n --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
+    --prefix=kmp_i18n --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
   DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl
 )
 add_custom_command(
   OUTPUT  kmp_i18n_default.inc
   COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_ARCH} --prefix=kmp_i18n --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
+    --prefix=kmp_i18n --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
   DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl
 )
 
 # Set the -D definitions for all sources
+# UNICODE and _UNICODE are set in LLVM's CMake system.  They affect the
+# ittnotify code and should only be set when compiling ittnotify_static.c
+# on Windows (done below).
+# TODO: Fix the UNICODE usage in ittnotify code for Windows.
+remove_definitions(-DUNICODE -D_UNICODE)
 libomp_get_definitions_flags(LIBOMP_CONFIGURED_DEFINITIONS_FLAGS)
 add_definitions(${LIBOMP_CONFIGURED_DEFINITIONS_FLAGS})
 
@@ -51,43 +56,27 @@ set(LIBOMP_CFILES)
 set(LIBOMP_CXXFILES)
 set(LIBOMP_ASMFILES)
 if(${STUBS_LIBRARY})
-  set(LIBOMP_CFILES kmp_stub.c)
+  set(LIBOMP_CFILES kmp_stub.cpp)
 else()
-  # Get C files
-  set(LIBOMP_CFILES
-    kmp_alloc.c
-    kmp_atomic.c
-    kmp_csupport.c
-    kmp_debug.c
-    kmp_itt.c
-    kmp_environment.c
-    kmp_error.c
-    kmp_global.c
-    kmp_i18n.c
-    kmp_io.c
-    kmp_runtime.c
-    kmp_settings.c
-    kmp_str.c
-    kmp_tasking.c
-    kmp_taskq.c
-    kmp_threadprivate.c
-    kmp_utility.c
-  )
-  if(WIN32)
-    # Windows specific files
-    libomp_append(LIBOMP_CFILES z_Windows_NT_util.c)
-    libomp_append(LIBOMP_CFILES z_Windows_NT-586_util.c)
-    libomp_append(LIBOMP_ASMFILES z_Windows_NT-586_asm.asm) # Windows assembly file
-  else()
-    # Unix specific files
-    libomp_append(LIBOMP_CFILES z_Linux_util.c)
-    libomp_append(LIBOMP_CFILES kmp_gsupport.c)
-    libomp_append(LIBOMP_ASMFILES z_Linux_asm.s) # Unix assembly file
-  endif()
-  libomp_append(LIBOMP_CFILES thirdparty/ittnotify/ittnotify_static.c LIBOMP_USE_ITT_NOTIFY)
-  libomp_append(LIBOMP_CFILES kmp_debugger.c LIBOMP_USE_DEBUGGER)
   # Get C++ files
   set(LIBOMP_CXXFILES
+    kmp_alloc.cpp
+    kmp_atomic.cpp
+    kmp_csupport.cpp
+    kmp_debug.cpp
+    kmp_itt.cpp
+    kmp_environment.cpp
+    kmp_error.cpp
+    kmp_global.cpp
+    kmp_i18n.cpp
+    kmp_io.cpp
+    kmp_runtime.cpp
+    kmp_settings.cpp
+    kmp_str.cpp
+    kmp_tasking.cpp
+    kmp_taskq.cpp
+    kmp_threadprivate.cpp
+    kmp_utility.cpp
     kmp_barrier.cpp
     kmp_wait_release.cpp
     kmp_affinity.cpp
@@ -95,6 +84,19 @@ else()
     kmp_lock.cpp
     kmp_sched.cpp
   )
+  if(WIN32)
+    # Windows specific files
+    libomp_append(LIBOMP_CXXFILES z_Windows_NT_util.cpp)
+    libomp_append(LIBOMP_CXXFILES z_Windows_NT-586_util.cpp)
+    libomp_append(LIBOMP_ASMFILES z_Windows_NT-586_asm.asm) # Windows assembly file
+  else()
+    # Unix specific files
+    libomp_append(LIBOMP_CXXFILES z_Linux_util.cpp)
+    libomp_append(LIBOMP_CXXFILES kmp_gsupport.cpp)
+    libomp_append(LIBOMP_ASMFILES z_Linux_asm.s) # Unix assembly file
+  endif()
+  libomp_append(LIBOMP_CFILES thirdparty/ittnotify/ittnotify_static.c LIBOMP_USE_ITT_NOTIFY)
+  libomp_append(LIBOMP_CXXFILES kmp_debugger.cpp LIBOMP_USE_DEBUGGER)
   libomp_append(LIBOMP_CXXFILES kmp_stats.cpp LIBOMP_STATS)
   libomp_append(LIBOMP_CXXFILES kmp_stats_timing.cpp LIBOMP_STATS)
   if(${LIBOMP_OMP_VERSION} GREATER 40 OR ${LIBOMP_OMP_VERSION} EQUAL 40)
@@ -103,10 +105,11 @@ else()
   endif()
 endif()
 # Files common to stubs and normal library
-libomp_append(LIBOMP_CFILES kmp_ftn_cdecl.c)
-libomp_append(LIBOMP_CFILES kmp_ftn_extra.c)
-libomp_append(LIBOMP_CFILES kmp_version.c)
-libomp_append(LIBOMP_CFILES ompt-general.c IF_TRUE LIBOMP_OMPT_SUPPORT)
+libomp_append(LIBOMP_CXXFILES kmp_ftn_cdecl.cpp)
+libomp_append(LIBOMP_CXXFILES kmp_ftn_extra.cpp)
+libomp_append(LIBOMP_CXXFILES kmp_version.cpp)
+libomp_append(LIBOMP_CXXFILES ompt-general.cpp IF_TRUE LIBOMP_OMPT_SUPPORT)
+libomp_append(LIBOMP_CXXFILES tsan_annotations.cpp IF_TRUE LIBOMP_TSAN_SUPPORT)
 
 set(LIBOMP_SOURCE_FILES ${LIBOMP_CFILES} ${LIBOMP_CXXFILES} ${LIBOMP_ASMFILES})
 # For Windows, there is a resource file (.rc -> .res) that is also compiled
@@ -136,7 +139,9 @@ endif()
 
 # Add the OpenMP library
 libomp_get_ldflags(LIBOMP_CONFIGURED_LDFLAGS)
-add_library(omp SHARED ${LIBOMP_SOURCE_FILES})
+
+add_library(omp ${LIBOMP_LIBRARY_KIND} ${LIBOMP_SOURCE_FILES})
+
 set_target_properties(omp PROPERTIES
   PREFIX "" SUFFIX "" OUTPUT_NAME "${LIBOMP_LIB_FILE}"
   LINK_FLAGS "${LIBOMP_CONFIGURED_LDFLAGS}"
@@ -144,7 +149,11 @@ set_target_properties(omp PROPERTIES
 )
 
 # Get the library's location within the build tree for the unit tester
-get_target_property(LIBOMP_LIBRARY_DIR omp LIBRARY_OUTPUT_DIRECTORY)
+if(NOT WIN32)
+  get_target_property(LIBOMP_LIBRARY_DIR omp LIBRARY_OUTPUT_DIRECTORY)
+else()
+  get_target_property(LIBOMP_LIBRARY_DIR omp RUNTIME_OUTPUT_DIRECTORY)
+endif()
 if(NOT LIBOMP_LIBRARY_DIR)
   set(LIBOMP_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
   set(LIBOMP_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
@@ -156,9 +165,9 @@ endif()
 if(NOT WIN32)
   add_custom_command(TARGET omp POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E create_symlink ${LIBOMP_LIB_FILE}
-      libgomp${CMAKE_SHARED_LIBRARY_SUFFIX}
+      libgomp${LIBOMP_LIBRARY_SUFFIX}
     COMMAND ${CMAKE_COMMAND} -E create_symlink ${LIBOMP_LIB_FILE}
-      libiomp5${CMAKE_SHARED_LIBRARY_SUFFIX}
+      libiomp5${LIBOMP_LIBRARY_SUFFIX}
     WORKING_DIRECTORY ${LIBOMP_LIBRARY_DIR}
   )
 endif()
@@ -181,16 +190,17 @@ if(WIN32)
   add_dependencies(omp libomp-needed-windows-files)
   # z_Windows_NT-586_asm.asm requires definitions to be sent via command line
   # It only needs the architecutre macro and OMPT_SUPPORT=0|1
-  libomp_append(LIBOMP_MASM_DEFINITIONS "_M_IA32" IF_TRUE IA32)
-  libomp_append(LIBOMP_MASM_DEFINITIONS "_M_AMD64" IF_TRUE INTEL64)
-  libomp_append(LIBOMP_MASM_DEFINITIONS "OMPT_SUPPORT" IF_TRUE_1_0 LIBOMP_OMPT_SUPPORT)
-  set_source_files_properties(z_Windows_NT-586_asm.asm PROPERTIES COMPILE_DEFINITIONS "${LIBOMP_MASM_DEFINITIONS}")
+  libomp_append(LIBOMP_MASM_DEFINITIONS "-D_M_IA32" IF_TRUE IA32)
+  libomp_append(LIBOMP_MASM_DEFINITIONS "-D_M_AMD64" IF_TRUE INTEL64)
+  libomp_append(LIBOMP_MASM_DEFINITIONS "-DOMPT_SUPPORT" IF_TRUE_1_0 LIBOMP_OMPT_SUPPORT)
+  libomp_list_to_string("${LIBOMP_MASM_DEFINITIONS}" LIBOMP_MASM_DEFINITIONS)
+  set_property(SOURCE z_Windows_NT-586_asm.asm APPEND_STRING PROPERTY COMPILE_FLAGS " ${LIBOMP_MASM_DEFINITIONS}")
   set_source_files_properties(thirdparty/ittnotify/ittnotify_static.c PROPERTIES COMPILE_DEFINITIONS "UNICODE")
 
   # Create Windows import library
-  # the import library is "re-linked" to include kmp_import.c which prevents
+  # the import library is "re-linked" to include kmp_import.cpp which prevents
   # linking of both Visual Studio OpenMP and newly built OpenMP
-  set_source_files_properties(kmp_import.c PROPERTIES COMPILE_FLAGS "${LIBOMP_CONFIGURED_CFLAGS}")
+  set_source_files_properties(kmp_import.cpp PROPERTIES COMPILE_FLAGS "${LIBOMP_CONFIGURED_CFLAGS}")
   set(LIBOMP_IMP_LIB_FILE ${LIBOMP_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
   set(LIBOMP_GENERATED_IMP_LIB_FILENAME ${LIBOMP_LIB_FILE}${CMAKE_STATIC_LIBRARY_SUFFIX})
   set_target_properties(omp PROPERTIES
@@ -206,8 +216,8 @@ if(WIN32)
     set(LIBOMP_GENERATED_IMP_LIB ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_GENERATED_IMP_LIB_FILENAME})
   endif()
   set_source_files_properties(${LIBOMP_GENERATED_IMP_LIB} PROPERTIES GENERATED TRUE EXTERNAL_OBJECT TRUE)
-  # Create new import library that is just the previously created one + kmp_import.c
-  add_library(ompimp STATIC ${LIBOMP_GENERATED_IMP_LIB} kmp_import.c)
+  # Create new import library that is just the previously created one + kmp_import.cpp
+  add_library(ompimp STATIC ${LIBOMP_GENERATED_IMP_LIB} kmp_import.cpp)
   set_target_properties(ompimp PROPERTIES
     PREFIX "" SUFFIX "" OUTPUT_NAME "${LIBOMP_IMP_LIB_FILE}"
     LINKER_LANGUAGE C
@@ -285,19 +295,23 @@ if(WIN32)
   set(LIBOMP_ALIASES "libiomp5md")
   foreach(alias IN LISTS LIBOMP_ALIASES)
     install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E copy \"${LIBOMP_LIB_FILE}\"
-      \"${alias}${CMAKE_SHARED_LIBRARY_SUFFIX}\" WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/bin)")
+      \"${alias}${LIBOMP_LIBRARY_SUFFIX}\" WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/bin)")
     install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E copy \"${LIBOMP_IMP_LIB_FILE}\"
-      \"${alias}${CMAKE_STATIC_LIBRARY_SUFFIX}\" WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib${LIBOMP_LIBDIR_SUFFIX})")
+      \"${alias}${LIBOMP_LIBRARY_SUFFIX}\" WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/lib${LIBOMP_LIBDIR_SUFFIX})")
   endforeach()
 else()
-  install(TARGETS omp LIBRARY DESTINATION lib${LIBOMP_LIBDIR_SUFFIX})
-  # Create aliases (symlinks) of the library for backwards compatibility
-  set(LIBOMP_ALIASES "libgomp;libiomp5")
-  foreach(alias IN LISTS LIBOMP_ALIASES)
-    install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink \"${LIBOMP_LIB_FILE}\"
-      \"${alias}${CMAKE_SHARED_LIBRARY_SUFFIX}\" WORKING_DIRECTORY
-      \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib${LIBOMP_LIBDIR_SUFFIX})")
-  endforeach()
+
+  install(TARGETS omp ${LIBOMP_INSTALL_KIND} DESTINATION lib${LIBOMP_LIBDIR_SUFFIX})
+
+  if(${LIBOMP_INSTALL_ALIASES})
+    # Create aliases (symlinks) of the library for backwards compatibility
+    set(LIBOMP_ALIASES "libgomp;libiomp5")
+    foreach(alias IN LISTS LIBOMP_ALIASES)
+      install(CODE "execute_process(COMMAND \"\${CMAKE_COMMAND}\" -E create_symlink \"${LIBOMP_LIB_FILE}\"
+        \"${alias}${LIBOMP_LIBRARY_SUFFIX}\" WORKING_DIRECTORY
+        \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib${LIBOMP_LIBDIR_SUFFIX})")
+    endforeach()
+  endif()
 endif()
 install(
   FILES

runtime/src/dllexports

@@ -162,9 +162,11 @@
         __kmp_wait_yield_4
         __kmp_fork_call
         __kmp_invoke_microtask
+    %ifdef KMP_USE_MONITOR
         __kmp_launch_monitor
-        __kmp_launch_worker
         __kmp_reap_monitor
+    %endif
+        __kmp_launch_worker
         __kmp_reap_worker
         __kmp_acquire_tas_lock
         __kmp_acquire_nested_tas_lock
@@ -184,21 +186,17 @@
     %ifdef USE_DEBUGGER
         __kmp_debugging                         DATA
         __kmp_omp_debug_struct_info             DATA
-    %endif 
+    %endif
 
-        # Symbols for MS mutual detection:
-        _You_must_link_with_exactly_one_OpenMP_library    DATA
-        _You_must_link_with_Intel_OpenMP_library          DATA
-        %ifdef msvc_compat
-            _You_must_link_with_Microsoft_OpenMP_library  DATA
-        %endif
+    # Symbols for MS mutual detection:
+    _You_must_link_with_exactly_one_OpenMP_library    DATA
+    _You_must_link_with_Intel_OpenMP_library          DATA
+    %ifdef msvc_compat
+        _You_must_link_with_Microsoft_OpenMP_library  DATA
+    %endif
 
-        __kmp_wait_32
-        __kmp_wait_64
-        __kmp_wait_oncore
-        __kmp_release_32
-        __kmp_release_64
-        __kmp_release_oncore
+    __kmp_wait_64
+    __kmp_release_64
 
 
 #    VT_getthid                              1
@@ -384,13 +382,20 @@ kmpc_set_defaults                           224
     %endif # OMP_40
 %endif
 
-# OpenMP 4.1 entry points
+# OpenMP 4.5 entry points
 %ifndef stub
-    %ifdef OMP_41
-	__kmpc_proxy_task_completed	    259
-	__kmpc_proxy_task_completed_ooo	    260
+    %ifdef OMP_45
+        __kmpc_proxy_task_completed         259
+        __kmpc_proxy_task_completed_ooo     260
+        __kmpc_doacross_init                261
+        __kmpc_doacross_wait                262
+        __kmpc_doacross_post                263
+        __kmpc_doacross_fini                264
+        __kmpc_taskloop                     266
     %endif
 %endif
+kmpc_aligned_malloc                         265
+kmpc_set_disp_num_buffers                   267
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
 # Number for lowercase version is indicated.  Number for uppercase is obtained by adding 1000.
@@ -443,6 +448,7 @@ kmp_get_num_known_threads                   743
 kmp_set_stacksize_s                         744
 kmp_get_stacksize_s                         745
 kmp_set_defaults                            746
+kmp_aligned_malloc                          747
 kmp_set_warnings_on                         779
 kmp_set_warnings_off                        780
 
@@ -490,14 +496,38 @@ kmp_set_warnings_off                        780
     omp_get_cancellation                    867
     kmp_get_cancellation_status             868
     omp_is_initial_device                   869
+    omp_set_default_device                  879
+    omp_get_default_device                  880
+    %ifdef stub
+        omp_get_num_devices                 881
+    %endif
 %endif # OMP_40
 
-# OpenMP 41
+# OpenMP 45
 
-%ifdef OMP_41
+%ifdef OMP_45
     omp_init_lock_with_hint                 870
     omp_init_nest_lock_with_hint            871
-%endif # OMP_41
+    omp_get_max_task_priority               872
+    omp_get_num_places                      873
+    omp_get_place_num_procs                 874
+    omp_get_place_proc_ids                  875
+    omp_get_place_num                       876
+    omp_get_partition_num_places            877
+    omp_get_partition_place_nums            878
+    %ifdef stub
+        omp_get_initial_device              882
+        omp_target_alloc                    883
+        omp_target_free                     884
+        omp_target_is_present               885
+        omp_target_memcpy                   886
+        omp_target_memcpy_rect              887
+        omp_target_associate_ptr            888
+        omp_target_disassociate_ptr         889
+    %endif
+%endif # OMP_45
+
+kmp_set_disp_num_buffers                    890
 
 %ifndef stub
     # Ordinals between 900 and 999 are reserved
@@ -894,6 +924,59 @@ kmp_set_warnings_off                        780
         __kmpc_atomic_start                    2410
         __kmpc_atomic_end                      2411
 
+        %ifdef HAVE_QUAD
+        __kmpc_atomic_fixed1_add_cpt_fp
+        __kmpc_atomic_fixed1_sub_cpt_fp
+        __kmpc_atomic_fixed1_mul_cpt_fp
+        __kmpc_atomic_fixed1_div_cpt_fp
+        __kmpc_atomic_fixed1u_add_cpt_fp
+        __kmpc_atomic_fixed1u_sub_cpt_fp
+        __kmpc_atomic_fixed1u_mul_cpt_fp
+        __kmpc_atomic_fixed1u_div_cpt_fp
+
+        __kmpc_atomic_fixed2_add_cpt_fp
+        __kmpc_atomic_fixed2_sub_cpt_fp
+        __kmpc_atomic_fixed2_mul_cpt_fp
+        __kmpc_atomic_fixed2_div_cpt_fp
+        __kmpc_atomic_fixed2u_add_cpt_fp
+        __kmpc_atomic_fixed2u_sub_cpt_fp
+        __kmpc_atomic_fixed2u_mul_cpt_fp
+        __kmpc_atomic_fixed2u_div_cpt_fp
+
+        __kmpc_atomic_fixed4_add_cpt_fp
+        __kmpc_atomic_fixed4_sub_cpt_fp
+        __kmpc_atomic_fixed4_mul_cpt_fp
+        __kmpc_atomic_fixed4_div_cpt_fp
+        __kmpc_atomic_fixed4u_add_cpt_fp
+        __kmpc_atomic_fixed4u_sub_cpt_fp
+        __kmpc_atomic_fixed4u_mul_cpt_fp
+        __kmpc_atomic_fixed4u_div_cpt_fp
+
+        __kmpc_atomic_fixed8_add_cpt_fp
+        __kmpc_atomic_fixed8_sub_cpt_fp
+        __kmpc_atomic_fixed8_mul_cpt_fp
+        __kmpc_atomic_fixed8_div_cpt_fp
+        __kmpc_atomic_fixed8u_add_cpt_fp
+        __kmpc_atomic_fixed8u_sub_cpt_fp
+        __kmpc_atomic_fixed8u_mul_cpt_fp
+        __kmpc_atomic_fixed8u_div_cpt_fp
+
+        __kmpc_atomic_float4_add_cpt_fp
+        __kmpc_atomic_float4_sub_cpt_fp
+        __kmpc_atomic_float4_mul_cpt_fp
+        __kmpc_atomic_float4_div_cpt_fp
+
+        __kmpc_atomic_float8_add_cpt_fp
+        __kmpc_atomic_float8_sub_cpt_fp
+        __kmpc_atomic_float8_mul_cpt_fp
+        __kmpc_atomic_float8_div_cpt_fp
+
+        __kmpc_atomic_float10_add_cpt_fp
+        __kmpc_atomic_float10_sub_cpt_fp
+        __kmpc_atomic_float10_mul_cpt_fp
+        __kmpc_atomic_float10_div_cpt_fp
+        %endif
+
         %ifdef OMP_40
 
             # ATOMIC extensions for OpenMP 4.0 spec (x86 and x64 only)
@@ -972,11 +1055,120 @@ kmp_set_warnings_off                        780
             %endif
             %endif
 
+            __kmpc_atomic_fixed1_sub_rev   2470
+            __kmpc_atomic_fixed1_div_rev   2471
+            __kmpc_atomic_fixed1u_div_rev  2472
+            __kmpc_atomic_fixed1_shl_rev   2473
+            __kmpc_atomic_fixed1_shr_rev   2474
+            __kmpc_atomic_fixed1u_shr_rev  2475
+            __kmpc_atomic_fixed2_sub_rev   2476
+            __kmpc_atomic_fixed2_div_rev   2477
+            __kmpc_atomic_fixed2u_div_rev  2478
+            __kmpc_atomic_fixed2_shl_rev   2479
+            __kmpc_atomic_fixed2_shr_rev   2480
+            __kmpc_atomic_fixed2u_shr_rev  2481
+            __kmpc_atomic_fixed4_sub_rev   2482
+            __kmpc_atomic_fixed4_div_rev   2483
+            __kmpc_atomic_fixed4u_div_rev  2484
+            __kmpc_atomic_fixed4_shl_rev   2485
+            __kmpc_atomic_fixed4_shr_rev   2486
+            __kmpc_atomic_fixed4u_shr_rev  2487
+            __kmpc_atomic_fixed8_sub_rev   2488
+            __kmpc_atomic_fixed8_div_rev   2489
+            __kmpc_atomic_fixed8u_div_rev  2490
+            __kmpc_atomic_fixed8_shl_rev   2491
+            __kmpc_atomic_fixed8_shr_rev   2492
+            __kmpc_atomic_fixed8u_shr_rev  2493
+            __kmpc_atomic_float4_sub_rev   2494
+            __kmpc_atomic_float4_div_rev   2495
+            __kmpc_atomic_float8_sub_rev   2496
+            __kmpc_atomic_float8_div_rev   2497
+            __kmpc_atomic_float10_sub_rev  2498
+            __kmpc_atomic_float10_div_rev  2499
+            %ifdef HAVE_QUAD
+            __kmpc_atomic_float16_sub_rev  2500
+            __kmpc_atomic_float16_div_rev  2501
+	    %endif
+            __kmpc_atomic_cmplx4_sub_rev   2502
+            __kmpc_atomic_cmplx4_div_rev   2503
+            __kmpc_atomic_cmplx8_sub_rev   2504
+            __kmpc_atomic_cmplx8_div_rev   2505
+            __kmpc_atomic_cmplx10_sub_rev  2506
+            __kmpc_atomic_cmplx10_div_rev  2507
+            %ifdef HAVE_QUAD
+            __kmpc_atomic_cmplx16_sub_rev  2508
+            __kmpc_atomic_cmplx16_div_rev  2509
+            %ifdef arch_32
+                __kmpc_atomic_float16_sub_a16_rev  2510
+                __kmpc_atomic_float16_div_a16_rev  2511
+                __kmpc_atomic_cmplx16_sub_a16_rev  2512
+                __kmpc_atomic_cmplx16_div_a16_rev  2513
+            %endif
+            %endif
+
+            %ifdef HAVE_QUAD
+            __kmpc_atomic_fixed1_sub_rev_fp
+            __kmpc_atomic_fixed1_div_rev_fp
+            __kmpc_atomic_fixed1u_div_rev_fp
+            __kmpc_atomic_fixed2_sub_rev_fp
+            __kmpc_atomic_fixed2_div_rev_fp
+            __kmpc_atomic_fixed2u_div_rev_fp
+            __kmpc_atomic_fixed4_sub_rev_fp
+            __kmpc_atomic_fixed4_div_rev_fp
+            __kmpc_atomic_fixed4u_div_rev_fp
+            __kmpc_atomic_fixed8_sub_rev_fp
+            __kmpc_atomic_fixed8_div_rev_fp
+            __kmpc_atomic_fixed8u_div_rev_fp
+            __kmpc_atomic_float4_sub_rev_fp
+            __kmpc_atomic_float4_div_rev_fp
+            __kmpc_atomic_float8_sub_rev_fp
+            __kmpc_atomic_float8_div_rev_fp
+            __kmpc_atomic_float10_sub_rev_fp
+            __kmpc_atomic_float10_div_rev_fp
+
+            __kmpc_atomic_fixed1_sub_cpt_rev_fp
+            __kmpc_atomic_fixed1u_sub_cpt_rev_fp
+            __kmpc_atomic_fixed1_div_cpt_rev_fp
+            __kmpc_atomic_fixed1u_div_cpt_rev_fp
+            __kmpc_atomic_fixed2_sub_cpt_rev_fp
+            __kmpc_atomic_fixed2u_sub_cpt_rev_fp
+            __kmpc_atomic_fixed2_div_cpt_rev_fp
+            __kmpc_atomic_fixed2u_div_cpt_rev_fp
+            __kmpc_atomic_fixed4_sub_cpt_rev_fp
+            __kmpc_atomic_fixed4u_sub_cpt_rev_fp
+            __kmpc_atomic_fixed4_div_cpt_rev_fp
+            __kmpc_atomic_fixed4u_div_cpt_rev_fp
+            __kmpc_atomic_fixed8_sub_cpt_rev_fp
+            __kmpc_atomic_fixed8u_sub_cpt_rev_fp
+            __kmpc_atomic_fixed8_div_cpt_rev_fp
+            __kmpc_atomic_fixed8u_div_cpt_rev_fp
+            __kmpc_atomic_float4_sub_cpt_rev_fp
+            __kmpc_atomic_float4_div_cpt_rev_fp
+            __kmpc_atomic_float8_sub_cpt_rev_fp
+            __kmpc_atomic_float8_div_cpt_rev_fp
+            __kmpc_atomic_float10_sub_cpt_rev_fp
+            __kmpc_atomic_float10_div_cpt_rev_fp
+            %endif
         %endif   # OMP_40
 
 
     %endif   # arch_64
 
+    %ifdef HAVE_QUAD
+    __kmpc_atomic_fixed1u_add_fp
+    __kmpc_atomic_fixed1u_sub_fp
+    __kmpc_atomic_fixed1u_mul_fp
+    __kmpc_atomic_fixed2u_add_fp
+    __kmpc_atomic_fixed2u_sub_fp
+    __kmpc_atomic_fixed2u_mul_fp
+    __kmpc_atomic_fixed4u_add_fp
+    __kmpc_atomic_fixed4u_sub_fp
+    __kmpc_atomic_fixed4u_mul_fp
+    __kmpc_atomic_fixed8u_add_fp
+    __kmpc_atomic_fixed8u_sub_fp
+    __kmpc_atomic_fixed8u_mul_fp
+    %endif
+
 %endif
 
 # end of file #

runtime/src/exports_so.txt

@@ -27,7 +27,10 @@ VERSION {
         #
         ompt_tool;           # OMPT initialization interface
         ompt_control;        # OMPT control interface
-        
+
+        # icc drops weak attribute at linking step without the following line:
+        Annotate*;           # TSAN annotation
+
         #
         # OMPT state placeholders
         #
@@ -85,16 +88,14 @@ VERSION {
         __kmp_invoke_microtask;
         __kmp_itt_fini_ittlib;
         __kmp_itt_init_ittlib;
+#if KMP_USE_MONITOR
         __kmp_launch_monitor;
-        __kmp_launch_worker;
         __kmp_reap_monitor;
+#endif
+        __kmp_launch_worker;
         __kmp_reap_worker;
-        __kmp_release_32;
         __kmp_release_64;
-        __kmp_release_oncore;
-        __kmp_wait_32;
         __kmp_wait_64;
-        __kmp_wait_oncore;
         __kmp_wait_yield_4;
 
         # ittnotify symbols to be used by debugger

runtime/src/i18n/en_US.txt

@@ -38,7 +38,7 @@ Language "English"
 Country  "USA"
 LangId   "1033"
 Version  "2"
-Revision "20140827"
+Revision "20160714"
 
 
 
@@ -388,10 +388,10 @@ OBSOLETE                     "%1$s: granularity=core will be used."
 EnvLockWarn                  "%1$s must be set prior to first OMP lock call or critical section; ignored."
 FutexNotSupported            "futex system call not supported; %1$s=%2$s ignored."
 AffGranUsing                 "%1$s: granularity=%2$s will be used."
-AffThrPlaceInvalid           "%1$s: invalid value \"%2$s\", valid format is \"Ns[@N],Nc[@N],Nt "
+AffHWSubsetInvalid           "%1$s: invalid value \"%2$s\", valid format is \"Ns[@N],Nc[@N],Nt "
                              "(nSockets@offset, nCores@offset, nTthreads per core)\"."
-AffThrPlaceUnsupported       "KMP_PLACE_THREADS ignored: unsupported architecture."
-AffThrPlaceManyCores         "KMP_PLACE_THREADS ignored: too many cores requested."
+AffHWSubsetUnsupported       "KMP_HW_SUBSET ignored: unsupported architecture."
+AffHWSubsetManyCores         "KMP_HW_SUBSET ignored: too many cores requested."
 SyntaxErrorUsing             "%1$s: syntax error, using %2$s."
 AdaptiveNotSupported         "%1$s: Adaptive locks are not supported; using queuing."
 EnvSyntaxError               "%1$s: Invalid symbols found. Check the value \"%2$s\"."
@@ -399,15 +399,18 @@ EnvSpacesNotAllowed          "%1$s: Spaces between digits are not allowed \"%2$s
 BoundToOSProcSet             "%1$s: pid %2$d thread %3$d bound to OS proc set %4$s"
 CnsLoopIncrIllegal           "%1$s error: parallel loop increment and condition are inconsistent."
 NoGompCancellation           "libgomp cancellation is not currently supported."
-AffThrPlaceNonUniform        "KMP_PLACE_THREADS ignored: non-uniform topology."
-AffThrPlaceNonThreeLevel     "KMP_PLACE_THREADS ignored: only three-level topology is supported."
+AffHWSubsetNonUniform        "KMP_HW_SUBSET ignored: non-uniform topology."
+AffHWSubsetNonThreeLevel     "KMP_HW_SUBSET ignored: only three-level topology is supported."
 AffGranTopGroup              "%1$s: granularity=%2$s is not supported with KMP_TOPOLOGY_METHOD=group. Using \"granularity=fine\"."
 AffGranGroupType             "%1$s: granularity=group is not supported with KMP_AFFINITY=%2$s. Using \"granularity=core\"."
-AffThrPlaceManySockets       "KMP_PLACE_THREADS ignored: too many sockets requested."
-AffThrPlaceDeprecated        "KMP_PLACE_THREADS \"o\" offset designator deprecated, please use @ prefix for offset value."
+AffHWSubsetManySockets       "KMP_HW_SUBSET ignored: too many sockets requested."
+AffHWSubsetDeprecated        "KMP_HW_SUBSET \"o\" offset designator deprecated, please use @ prefix for offset value."
 AffUsingHwloc                "%1$s: Affinity capable, using hwloc."
 AffIgnoringHwloc             "%1$s: Ignoring hwloc mechanism."
 AffHwlocErrorOccurred        "%1$s: Hwloc failed in %2$s. Relying on internal affinity mechanisms."
+EnvSerialWarn                "%1$s must be set prior to OpenMP runtime library initialization; ignored."
+EnvVarDeprecated             "%1$s variable deprecated, please use %2$s instead."
+RedMethodNotSupported        "KMP_FORCE_REDUCTION: %1$s method is not supported; using critical."
 
 
 # --------------------------------------------------------------------------------------------------

runtime/src/include/30/omp.h.var

@@ -141,6 +141,7 @@
     extern int    __KAI_KMPC_CONVENTION  kmp_get_affinity_mask_proc   (int, kmp_affinity_mask_t *);
 
     extern void * __KAI_KMPC_CONVENTION  kmp_malloc  (size_t);
+    extern void * __KAI_KMPC_CONVENTION  kmp_aligned_malloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_calloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_realloc (void *, size_t);
     extern void   __KAI_KMPC_CONVENTION  kmp_free    (void *);

runtime/src/include/30/omp_lib.f.var

@@ -323,6 +323,13 @@
             integer (kind=kmp_size_t_kind) size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind) size
+            integer (kind=kmp_size_t_kind) alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc
@@ -416,6 +423,7 @@
 !dec$ attributes alias:'KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'KMP_FREE'::kmp_free
@@ -482,6 +490,7 @@
 !dec$ attributes alias:'_KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'_KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'_KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'_KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'_KMP_FREE'::kmp_free
@@ -551,6 +560,7 @@
 !dec$ attributes alias:'kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'kmp_free_'::kmp_free
@@ -620,6 +630,7 @@
 !dec$ attributes alias:'_kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'_kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'_kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'_kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'_kmp_free_'::kmp_free

runtime/src/include/30/omp_lib.f90.var

@@ -328,6 +328,13 @@
             integer (kind=kmp_size_t_kind), value :: size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment) bind(c)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind), value :: size
+            integer (kind=kmp_size_t_kind), value :: alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc

runtime/src/include/30/omp_lib.h.var

@@ -321,6 +321,13 @@
           integer (kind=kmp_size_t_kind) size
         end function kmp_malloc
 
+        function kmp_aligned_malloc(size, alignment)
+          import
+          integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+          integer (kind=kmp_size_t_kind) size
+          integer (kind=kmp_size_t_kind) alignment
+        end function kmp_aligned_malloc
+
         function kmp_calloc(nelem, elsize)
           import
           integer (kind=kmp_pointer_kind) kmp_calloc
@@ -416,6 +423,7 @@
 !dec$ attributes alias:'KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'KMP_FREE'::kmp_free
@@ -484,6 +492,7 @@
 !dec$ attributes alias:'_KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'_KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'_KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'_KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'_KMP_FREE'::kmp_free
@@ -555,6 +564,7 @@
 !dec$ attributes alias:'kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'kmp_free_'::kmp_free
@@ -626,6 +636,7 @@
 !dec$ attributes alias:'_kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'_kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'_kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'_kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'_kmp_free_'::kmp_free

runtime/src/include/30/ompt.h.var

@@ -305,7 +305,7 @@ typedef void (*ompt_new_parallel_callback_t) (
 typedef void (*ompt_end_parallel_callback_t) (
     ompt_parallel_id_t parallel_id,   /* id of parallel region       */
     ompt_task_id_t task_id,           /* id of task                  */
-    ompt_invoker_t invoker            /* who invokes master task?    */ 
+    ompt_invoker_t invoker            /* who invokes master task?    */
 );
 
 /* tasks */
@@ -461,14 +461,15 @@ OMPT_API_FUNCTION(int, ompt_get_callback, (
  ***************************************************************************/
 
 /* control */
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+// FIXME: remove workaround for clang
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp declare target
 #endif
 void ompt_control(
     uint64_t command,
     uint64_t modifier
 );
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp end declare target
 #endif
 

runtime/src/include/40/omp.h.var

@@ -137,6 +137,7 @@
     extern omp_proc_bind_t __KAI_KMPC_CONVENTION omp_get_proc_bind (void);
 
     extern void * __KAI_KMPC_CONVENTION  kmp_malloc  (size_t);
+    extern void * __KAI_KMPC_CONVENTION  kmp_aligned_malloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_calloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_realloc (void *, size_t);
     extern void   __KAI_KMPC_CONVENTION  kmp_free    (void *);

runtime/src/include/40/omp_lib.f.var

@@ -66,19 +66,19 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(nthreads)
+          subroutine omp_set_num_threads(num_threads)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) nthreads
+            integer (kind=omp_integer_kind) num_threads
           end subroutine omp_set_num_threads
 
-          subroutine omp_set_dynamic(enable)
+          subroutine omp_set_dynamic(dynamic_threads)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind) enable
+            logical (kind=omp_logical_kind) dynamic_threads
           end subroutine omp_set_dynamic
 
-          subroutine omp_set_nested(enable)
+          subroutine omp_set_nested(nested)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind) enable
+            logical (kind=omp_logical_kind) nested
           end subroutine omp_set_nested
 
           function omp_get_num_threads()
@@ -106,6 +106,11 @@
             logical (kind=omp_logical_kind) omp_in_parallel
           end function omp_in_parallel
 
+          function omp_in_final()
+            use omp_lib_kinds
+            logical (kind=omp_logical_kind) omp_in_final
+          end function omp_in_final
+
           function omp_get_dynamic()
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_get_dynamic
@@ -153,16 +158,16 @@
             integer (kind=omp_integer_kind) omp_get_team_size
           end function omp_get_team_size
 
-          subroutine omp_set_schedule(kind, modifier)
+          subroutine omp_set_schedule(kind, chunk_size)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_set_schedule
 
-          subroutine omp_get_schedule(kind, modifier)
+          subroutine omp_get_schedule(kind, chunk_size)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_get_schedule
 
           function omp_get_proc_bind()
@@ -183,9 +188,9 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(dflt_device)
+          subroutine omp_set_default_device(device_num)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) dflt_device
+            integer (kind=omp_integer_kind) device_num
           end subroutine omp_set_default_device
 
           function omp_get_num_devices()
@@ -213,86 +218,86 @@
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
-          subroutine omp_init_lock(lockvar)
+          subroutine omp_init_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 
-          subroutine omp_destroy_lock(lockvar)
+          subroutine omp_destroy_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_destroy_lock
 
-          subroutine omp_set_lock(lockvar)
+          subroutine omp_set_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_set_lock
 
-          subroutine omp_unset_lock(lockvar)
+          subroutine omp_unset_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_unset_lock
 
-          function omp_test_lock(lockvar)
+          function omp_test_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_test_lock
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end function omp_test_lock
 
-          subroutine omp_init_nest_lock(lockvar)
+          subroutine omp_init_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_init_nest_lock
 
-          subroutine omp_destroy_nest_lock(lockvar)
+          subroutine omp_destroy_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_destroy_nest_lock
 
-          subroutine omp_set_nest_lock(lockvar)
+          subroutine omp_set_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_set_nest_lock
 
-          subroutine omp_unset_nest_lock(lockvar)
+          subroutine omp_unset_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_unset_nest_lock
 
-          function omp_test_nest_lock(lockvar)
+          function omp_test_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_test_nest_lock
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end function omp_test_nest_lock
 
 !         ***
@@ -406,6 +411,13 @@
             integer (kind=kmp_size_t_kind) size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind) size
+            integer (kind=kmp_size_t_kind) alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc
@@ -513,6 +525,7 @@
 !dec$ attributes alias:'KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'KMP_FREE'::kmp_free
@@ -589,6 +602,7 @@
 !dec$ attributes alias:'_KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'_KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'_KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'_KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'_KMP_FREE'::kmp_free
@@ -668,6 +682,7 @@
 !dec$ attributes alias:'kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'kmp_free_'::kmp_free
@@ -743,6 +758,7 @@
 !dec$ attributes alias:'_kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'_kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'_kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'_kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'_kmp_free_'::kmp_free

runtime/src/include/40/omp_lib.f90.var

@@ -64,19 +64,19 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(nthreads) bind(c)
+          subroutine omp_set_num_threads(num_threads) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind), value :: nthreads
+            integer (kind=omp_integer_kind), value :: num_threads
           end subroutine omp_set_num_threads
 
-          subroutine omp_set_dynamic(enable) bind(c)
+          subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind), value :: enable
+            logical (kind=omp_logical_kind), value :: dynamic_threads
           end subroutine omp_set_dynamic
 
-          subroutine omp_set_nested(enable) bind(c)
+          subroutine omp_set_nested(nested) bind(c)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind), value :: enable
+            logical (kind=omp_logical_kind), value :: nested
           end subroutine omp_set_nested
 
           function omp_get_num_threads() bind(c)
@@ -156,16 +156,16 @@
             integer (kind=omp_integer_kind), value :: level
           end function omp_get_team_size
 
-          subroutine omp_set_schedule(kind, modifier) bind(c)
+          subroutine omp_set_schedule(kind, chunk_size) bind(c)
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
-            integer (kind=omp_integer_kind), value :: modifier
+            integer (kind=omp_integer_kind), value :: chunk_size
           end subroutine omp_set_schedule
 
-          subroutine omp_get_schedule(kind, modifier) bind(c)
+          subroutine omp_get_schedule(kind, chunk_size) bind(c)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_get_schedule
 
           function omp_get_proc_bind() bind(c)
@@ -188,9 +188,9 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(dflt_device) bind(c)
+          subroutine omp_set_default_device(device_num) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind), value :: dflt_device
+            integer (kind=omp_integer_kind), value :: device_num
           end subroutine omp_set_default_device
 
           function omp_get_num_devices() bind(c)
@@ -218,86 +218,86 @@
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
-          subroutine omp_init_lock(lockvar) bind(c)
+          subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 
-          subroutine omp_destroy_lock(lockvar) bind(c)
+          subroutine omp_destroy_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_destroy_lock
 
-          subroutine omp_set_lock(lockvar) bind(c)
+          subroutine omp_set_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_set_lock
 
-          subroutine omp_unset_lock(lockvar) bind(c)
+          subroutine omp_unset_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_unset_lock
 
-          function omp_test_lock(lockvar) bind(c)
+          function omp_test_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_test_lock
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end function omp_test_lock
 
-          subroutine omp_init_nest_lock(lockvar) bind(c)
+          subroutine omp_init_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_init_nest_lock
 
-          subroutine omp_destroy_nest_lock(lockvar) bind(c)
+          subroutine omp_destroy_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_destroy_nest_lock
 
-          subroutine omp_set_nest_lock(lockvar) bind(c)
+          subroutine omp_set_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_set_nest_lock
 
-          subroutine omp_unset_nest_lock(lockvar) bind(c)
+          subroutine omp_unset_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_unset_nest_lock
 
-          function omp_test_nest_lock(lockvar) bind(c)
+          function omp_test_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_test_nest_lock
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end function omp_test_nest_lock
 
 !         ***
@@ -412,6 +412,13 @@
             integer (kind=kmp_size_t_kind), value :: size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment) bind(c)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind), value :: size
+            integer (kind=kmp_size_t_kind), value :: alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc

runtime/src/include/40/omp_lib.h.var

@@ -53,19 +53,19 @@
 !       *** omp_* entry points
 !       ***
 
-        subroutine omp_set_num_threads(nthreads) bind(c)
+        subroutine omp_set_num_threads(num_threads) bind(c)
           import
-          integer (kind=omp_integer_kind), value :: nthreads
+          integer (kind=omp_integer_kind), value :: num_threads
         end subroutine omp_set_num_threads
 
-        subroutine omp_set_dynamic(enable) bind(c)
+        subroutine omp_set_dynamic(dynamic_threads) bind(c)
           import
-          logical (kind=omp_logical_kind), value :: enable
+          logical (kind=omp_logical_kind), value :: dynamic_threads
         end subroutine omp_set_dynamic
 
-        subroutine omp_set_nested(enable) bind(c)
+        subroutine omp_set_nested(nested) bind(c)
           import
-          logical (kind=omp_logical_kind), value :: enable
+          logical (kind=omp_logical_kind), value :: nested
         end subroutine omp_set_nested
 
         function omp_get_num_threads() bind(c)
@@ -145,16 +145,16 @@
           integer (kind=omp_integer_kind), value :: level
         end function omp_get_team_size
 
-        subroutine omp_set_schedule(kind, modifier) bind(c)
+        subroutine omp_set_schedule(kind, chunk_size) bind(c)
           import
           integer (kind=omp_sched_kind), value :: kind
-          integer (kind=omp_integer_kind), value :: modifier
+          integer (kind=omp_integer_kind), value :: chunk_size
         end subroutine omp_set_schedule
 
-        subroutine omp_get_schedule(kind, modifier) bind(c)
+        subroutine omp_get_schedule(kind, chunk_size) bind(c)
           import
           integer (kind=omp_sched_kind) kind
-          integer (kind=omp_integer_kind) modifier
+          integer (kind=omp_integer_kind) chunk_size
         end subroutine omp_get_schedule
 
         function omp_get_proc_bind() bind(c)
@@ -175,9 +175,9 @@
           integer (kind=omp_integer_kind) omp_get_default_device
         end function omp_get_default_device
 
-        subroutine omp_set_default_device(dflt_device) bind(c)
+        subroutine omp_set_default_device(device_num) bind(c)
           import
-          integer (kind=omp_integer_kind), value :: dflt_device
+          integer (kind=omp_integer_kind), value :: device_num
         end subroutine omp_set_default_device
 
         function omp_get_num_devices() bind(c)
@@ -200,86 +200,86 @@
           logical (kind=omp_logical_kind) omp_is_initial_device
         end function omp_is_initial_device
 
-        subroutine omp_init_lock(lockvar) bind(c)
+        subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_init_lock
 
-        subroutine omp_destroy_lock(lockvar) bind(c)
+        subroutine omp_destroy_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_destroy_lock
 
-        subroutine omp_set_lock(lockvar) bind(c)
+        subroutine omp_set_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_set_lock
 
-        subroutine omp_unset_lock(lockvar) bind(c)
+        subroutine omp_unset_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_unset_lock
 
-        function omp_test_lock(lockvar) bind(c)
+        function omp_test_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
           import
           logical (kind=omp_logical_kind) omp_test_lock
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end function omp_test_lock
 
-        subroutine omp_init_nest_lock(lockvar) bind(c)
+        subroutine omp_init_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_init_nest_lock
 
-        subroutine omp_destroy_nest_lock(lockvar) bind(c)
+        subroutine omp_destroy_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_destroy_nest_lock
 
-        subroutine omp_set_nest_lock(lockvar) bind(c)
+        subroutine omp_set_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_set_nest_lock
 
-        subroutine omp_unset_nest_lock(lockvar) bind(c)
+        subroutine omp_unset_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_unset_nest_lock
 
-        function omp_test_nest_lock(lockvar) bind(c)
+        function omp_test_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
           import
           integer (kind=omp_integer_kind) omp_test_nest_lock
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end function omp_test_nest_lock
 
 !       ***
@@ -393,6 +393,13 @@
           integer (kind=kmp_size_t_kind), value :: size
         end function kmp_malloc
 
+        function kmp_aligned_malloc(size, alignment) bind(c)
+          import
+          integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+          integer (kind=kmp_size_t_kind), value :: size
+          integer (kind=kmp_size_t_kind), value :: alignment
+        end function kmp_aligned_malloc
+
         function kmp_calloc(nelem, elsize) bind(c)
           import
           integer (kind=kmp_pointer_kind) kmp_calloc
@@ -481,6 +488,7 @@
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_unset_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_malloc
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_aligned_malloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_calloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_realloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_free
@@ -548,6 +556,7 @@
 !$omp declare target(kmp_unset_affinity_mask_proc )
 !$omp declare target(kmp_get_affinity_mask_proc )
 !$omp declare target(kmp_malloc )
+!$omp declare target(kmp_aligned_malloc )
 !$omp declare target(kmp_calloc )
 !$omp declare target(kmp_realloc )
 !$omp declare target(kmp_free )

runtime/src/include/40/ompt.h.var

@@ -171,7 +171,10 @@
     macro (ompt_event_destroy_lock,             ompt_wait_callback_t,          59) /* lock destruction */       \
     macro (ompt_event_destroy_nest_lock,        ompt_wait_callback_t,          60) /* nest lock destruction */  \
                                                                                                                 \
-    macro (ompt_event_flush,                    ompt_callback_t,               61) /* after executing flush */
+    macro (ompt_event_flush,                    ompt_callback_t,               61) /* after executing flush */  \
+                                                                                                                \
+    macro (ompt_event_task_dependences,         ompt_task_dependences_callback_t, 69) /* report task dependences  */\
+    macro (ompt_event_task_dependence_pair,     ompt_task_pair_callback_t,     70) /* report task dependence pair */
 
 
 
@@ -206,6 +209,23 @@ typedef struct ompt_frame_s {
 } ompt_frame_t;
 
 
+/*---------------------
+ * dependences types
+ *---------------------*/
+
+typedef enum ompt_task_dependence_flag_e {
+    // a two bit field for the dependence type
+    ompt_task_dependence_type_out   = 1,
+    ompt_task_dependence_type_in    = 2,
+    ompt_task_dependence_type_inout = 3,
+} ompt_task_dependence_flag_t;
+
+typedef struct ompt_task_dependence_s {
+    void *variable_addr;
+    uint32_t  dependence_flags;
+} ompt_task_dependence_t;
+
+
 /*****************************************************************************
  * enumerations for thread states and runtime events
  *****************************************************************************/
@@ -305,7 +325,7 @@ typedef void (*ompt_new_parallel_callback_t) (
 typedef void (*ompt_end_parallel_callback_t) (
     ompt_parallel_id_t parallel_id,   /* id of parallel region       */
     ompt_task_id_t task_id,           /* id of task                  */
-    ompt_invoker_t invoker            /* who invokes master task?    */ 
+    ompt_invoker_t invoker            /* who invokes master task?    */
 );
 
 /* tasks */
@@ -325,6 +345,13 @@ typedef void (*ompt_new_task_callback_t) (
     void *task_function               /* pointer to outlined function */
 );
 
+/* task dependences */
+typedef void (*ompt_task_dependences_callback_t) (
+    ompt_task_id_t task_id,            /* ID of task with dependences */
+    const ompt_task_dependence_t *deps,/* vector of task dependences  */
+    int ndeps                          /* number of dependences       */
+);
+
 /* program */
 typedef void (*ompt_control_callback_t) (
     uint64_t command,                 /* command of control call      */
@@ -461,14 +488,15 @@ OMPT_API_FUNCTION(int, ompt_get_callback, (
  ***************************************************************************/
 
 /* control */
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+// FIXME: remove workaround for clang
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp declare target
 #endif
 void ompt_control(
     uint64_t command,
     uint64_t modifier
 );
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp end declare target
 #endif
 

runtime/src/include/45/omp.h.var

@@ -1,5 +1,5 @@
 /*
- * include/41/omp.h.var
+ * include/45/omp.h.var
  */
 
 
@@ -62,6 +62,7 @@
     extern int    __KAI_KMPC_CONVENTION  omp_get_thread_limit        (void);
     extern int    __KAI_KMPC_CONVENTION  omp_get_max_active_levels   (void);
     extern void   __KAI_KMPC_CONVENTION  omp_get_schedule            (omp_sched_t *, int *);
+    extern int    __KAI_KMPC_CONVENTION  omp_get_max_task_priority   (void);
 
     /* lock API functions */
     typedef struct omp_lock_t {
@@ -115,6 +116,17 @@
     extern int  __KAI_KMPC_CONVENTION  omp_get_cancellation (void);
 
 #   include <stdlib.h>
+    /* OpenMP 4.5 */
+    extern int   __KAI_KMPC_CONVENTION  omp_get_initial_device (void);
+    extern void* __KAI_KMPC_CONVENTION  omp_target_alloc(size_t, int);
+    extern void  __KAI_KMPC_CONVENTION  omp_target_free(void *, int);
+    extern int   __KAI_KMPC_CONVENTION  omp_target_is_present(void *, int);
+    extern int   __KAI_KMPC_CONVENTION  omp_target_memcpy(void *, void *, size_t, size_t, size_t, int, int);
+    extern int   __KAI_KMPC_CONVENTION  omp_target_memcpy_rect(void *, void *, size_t, int, const size_t *,
+                                            const size_t *, const size_t *, const size_t *, const size_t *, int, int);
+    extern int   __KAI_KMPC_CONVENTION  omp_target_associate_ptr(void *, void *, size_t, size_t, int);
+    extern int   __KAI_KMPC_CONVENTION  omp_target_disassociate_ptr(void *, int);
+
     /* kmp API functions */
     extern int    __KAI_KMPC_CONVENTION  kmp_get_stacksize          (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_stacksize          (int);
@@ -128,6 +140,7 @@
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library_turnaround (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_library_throughput (void);
     extern void   __KAI_KMPC_CONVENTION  kmp_set_defaults           (char const *);
+    extern void   __KAI_KMPC_CONVENTION  kmp_set_disp_num_buffers   (int);
 
     /* Intel affinity API */
     typedef void * kmp_affinity_mask_t;
@@ -152,7 +165,16 @@
 
     extern omp_proc_bind_t __KAI_KMPC_CONVENTION omp_get_proc_bind (void);
 
+    /* OpenMP 4.5 affinity API */
+    extern int  __KAI_KMPC_CONVENTION omp_get_num_places (void);
+    extern int  __KAI_KMPC_CONVENTION omp_get_place_num_procs (int);
+    extern void __KAI_KMPC_CONVENTION omp_get_place_proc_ids (int, int *);
+    extern int  __KAI_KMPC_CONVENTION omp_get_place_num (void);
+    extern int  __KAI_KMPC_CONVENTION omp_get_partition_num_places (void);
+    extern void __KAI_KMPC_CONVENTION omp_get_partition_place_nums (int *);
+
     extern void * __KAI_KMPC_CONVENTION  kmp_malloc  (size_t);
+    extern void * __KAI_KMPC_CONVENTION  kmp_aligned_malloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_calloc  (size_t, size_t);
     extern void * __KAI_KMPC_CONVENTION  kmp_realloc (void *, size_t);
     extern void   __KAI_KMPC_CONVENTION  kmp_free    (void *);

runtime/src/include/45/omp_lib.f.var

@@ -1,4 +1,4 @@
-! include/41/omp_lib.f.var
+! include/45/omp_lib.f.var
 
 !
 !//===----------------------------------------------------------------------===//
@@ -76,19 +76,19 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(nthreads)
+          subroutine omp_set_num_threads(num_threads)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) nthreads
+            integer (kind=omp_integer_kind) num_threads
           end subroutine omp_set_num_threads
 
-          subroutine omp_set_dynamic(enable)
+          subroutine omp_set_dynamic(dynamic_threads)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind) enable
+            logical (kind=omp_logical_kind) dynamic_threads
           end subroutine omp_set_dynamic
 
-          subroutine omp_set_nested(enable)
+          subroutine omp_set_nested(nested)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind) enable
+            logical (kind=omp_logical_kind) nested
           end subroutine omp_set_nested
 
           function omp_get_num_threads()
@@ -116,6 +116,11 @@
             logical (kind=omp_logical_kind) omp_in_parallel
           end function omp_in_parallel
 
+          function omp_in_final()
+            use omp_lib_kinds
+            logical (kind=omp_logical_kind) omp_in_final
+          end function omp_in_final
+
           function omp_get_dynamic()
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_get_dynamic
@@ -163,16 +168,16 @@
             integer (kind=omp_integer_kind) omp_get_team_size
           end function omp_get_team_size
 
-          subroutine omp_set_schedule(kind, modifier)
+          subroutine omp_set_schedule(kind, chunk_size)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_set_schedule
 
-          subroutine omp_get_schedule(kind, modifier)
+          subroutine omp_get_schedule(kind, chunk_size)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_get_schedule
 
           function omp_get_proc_bind()
@@ -180,6 +185,38 @@
             integer (kind=omp_proc_bind_kind) omp_get_proc_bind
           end function omp_get_proc_bind
 
+          function omp_get_num_places()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_num_places
+          end function omp_get_num_places
+
+          function omp_get_place_num_procs(place_num)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) place_num
+            integer (kind=omp_integer_kind) omp_get_place_num_procs
+          end function omp_get_place_num_procs
+
+          subroutine omp_get_place_proc_ids(place_num, ids)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) place_num
+            integer (kind=kmp_pointer_kind) ids(*)
+          end subroutine omp_get_place_proc_ids
+
+          function omp_get_place_num()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_place_num
+          end function omp_get_place_num
+
+          function omp_get_partition_num_places()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_partition_num_places
+          end function omp_get_partition_num_places
+
+          subroutine omp_get_partition_place_nums(place_nums)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) place_nums(*)
+          end subroutine omp_get_partition_place_nums
+
           function omp_get_wtime()
             double precision omp_get_wtime
           end function omp_get_wtime
@@ -193,9 +230,9 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(dflt_device)
+          subroutine omp_set_default_device(device_num)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) dflt_device
+            integer (kind=omp_integer_kind) device_num
           end subroutine omp_set_default_device
 
           function omp_get_num_devices()
@@ -223,88 +260,98 @@
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
-          subroutine omp_init_lock(lockvar)
+          function omp_get_initial_device()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_initial_device
+          end function omp_get_initial_device
+
+          subroutine omp_init_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 
-          subroutine omp_destroy_lock(lockvar)
+          subroutine omp_destroy_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_destroy_lock
 
-          subroutine omp_set_lock(lockvar)
+          subroutine omp_set_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_set_lock
 
-          subroutine omp_unset_lock(lockvar)
+          subroutine omp_unset_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_unset_lock
 
-          function omp_test_lock(lockvar)
+          function omp_test_lock(svar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_test_lock
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end function omp_test_lock
 
-          subroutine omp_init_nest_lock(lockvar)
+          subroutine omp_init_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_init_nest_lock
 
-          subroutine omp_destroy_nest_lock(lockvar)
+          subroutine omp_destroy_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_destroy_nest_lock
 
-          subroutine omp_set_nest_lock(lockvar)
+          subroutine omp_set_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_set_nest_lock
 
-          subroutine omp_unset_nest_lock(lockvar)
+          subroutine omp_unset_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_unset_nest_lock
 
-          function omp_test_nest_lock(lockvar)
+          function omp_test_nest_lock(nvar)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_test_nest_lock
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end function omp_test_nest_lock
 
+          function omp_get_max_task_priority()
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_max_task_priority
+          end function omp_get_max_task_priority
+
 !         ***
 !         *** kmp_* entry points
 !         ***
@@ -362,6 +409,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
+          subroutine kmp_set_disp_num_buffers(num)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) num
+          end subroutine kmp_set_disp_num_buffers
+
           function kmp_set_affinity(mask)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) kmp_set_affinity
@@ -416,6 +468,13 @@
             integer (kind=kmp_size_t_kind) size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind) size
+            integer (kind=kmp_size_t_kind) alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc
@@ -447,17 +506,17 @@
             logical (kind=omp_logical_kind) kmp_get_cancellation_status
           end function kmp_get_cancellation_status
 
-          subroutine kmp_init_lock_with_hint(lockvar, lockhint)
+          subroutine omp_init_lock_with_hint(svar, hint)
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
-            integer (kind=omp_lock_hint_kind) lockhint
-          end subroutine kmp_init_lock_with_hint
+            integer (kind=omp_lock_kind) svar
+            integer (kind=omp_lock_hint_kind) hint
+          end subroutine omp_init_lock_with_hint
 
-          subroutine kmp_init_nest_lock_with_hint(lockvar, lockhint)
+          subroutine omp_init_nest_lock_with_hint(nvar, hint)
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
-            integer (kind=omp_lock_hint_kind) lockhint
-          end subroutine kmp_init_nest_lock_with_hint
+            integer (kind=omp_nest_lock_kind) nvar
+            integer (kind=omp_lock_hint_kind) hint
+          end subroutine omp_init_nest_lock_with_hint
 
         end interface
 
@@ -503,6 +562,7 @@
 !dec$ attributes alias:'OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'OMP_GET_MAX_TASK_PRIORITY' :: omp_get_max_task_priority
 
 !dec$ attributes alias:'omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint' :: omp_init_lock_with_hint
@@ -537,6 +597,7 @@
 !dec$ attributes alias:'KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'KMP_FREE'::kmp_free
@@ -581,6 +642,7 @@
 !dec$ attributes alias:'_OMP_GET_TEAM_NUM' :: omp_get_team_num
 !dec$ attributes alias:'_OMP_GET_CANCELLATION' :: omp_get_cancellation
 !dec$ attributes alias:'_OMP_IS_INITIAL_DEVICE' :: omp_is_initial_device
+!dec$ attributes alias:'_OMP_GET_MAX_TASK_PRIORTY' :: omp_get_max_task_priority
 
 !dec$ attributes alias:'_omp_init_lock' :: omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint' :: omp_init_lock_with_hint
@@ -615,6 +677,7 @@
 !dec$ attributes alias:'_KMP_UNSET_AFFINITY_MASK_PROC'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_KMP_GET_AFFINITY_MASK_PROC'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_KMP_MALLOC'::kmp_malloc
+!dec$ attributes alias:'_KMP_ALIGNED_MALLOC'::kmp_aligned_malloc
 !dec$ attributes alias:'_KMP_CALLOC'::kmp_calloc
 !dec$ attributes alias:'_KMP_REALLOC'::kmp_realloc
 !dec$ attributes alias:'_KMP_FREE'::kmp_free
@@ -662,6 +725,7 @@
 !dec$ attributes alias:'omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'omp_get_max_task_priority_'::omp_get_max_task_priority
 
 !dec$ attributes alias:'omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'omp_init_lock_with_hint_'::omp_init_lock_with_hint
@@ -696,6 +760,7 @@
 !dec$ attributes alias:'kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'kmp_free_'::kmp_free
@@ -739,6 +804,7 @@
 !dec$ attributes alias:'_omp_get_team_num_'::omp_get_team_num
 !dec$ attributes alias:'_omp_get_cancellation_'::omp_get_cancellation
 !dec$ attributes alias:'_omp_is_initial_device_'::omp_is_initial_device
+!dec$ attributes alias:'_omp_get_max_task_priorty_'::omp_get_max_task_priority
 
 !dec$ attributes alias:'_omp_init_lock_'::omp_init_lock
 !dec$ attributes alias:'_omp_init_lock_with_hint_'::omp_init_lock_with_hint
@@ -773,6 +839,7 @@
 !dec$ attributes alias:'_kmp_unset_affinity_mask_proc_'::kmp_unset_affinity_mask_proc
 !dec$ attributes alias:'_kmp_get_affinity_mask_proc_'::kmp_get_affinity_mask_proc
 !dec$ attributes alias:'_kmp_malloc_'::kmp_malloc
+!dec$ attributes alias:'_kmp_aligned_malloc_'::kmp_aligned_malloc
 !dec$ attributes alias:'_kmp_calloc_'::kmp_calloc
 !dec$ attributes alias:'_kmp_realloc_'::kmp_realloc
 !dec$ attributes alias:'_kmp_free_'::kmp_free

runtime/src/include/45/omp_lib.f90.var

@@ -1,4 +1,4 @@
-! include/41/omp_lib.f90.var
+! include/45/omp_lib.f90.var
 
 !
 !//===----------------------------------------------------------------------===//
@@ -74,19 +74,19 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(nthreads) bind(c)
+          subroutine omp_set_num_threads(num_threads) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind), value :: nthreads
+            integer (kind=omp_integer_kind), value :: num_threads
           end subroutine omp_set_num_threads
 
-          subroutine omp_set_dynamic(enable) bind(c)
+          subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind), value :: enable
+            logical (kind=omp_logical_kind), value :: dynamic_threads
           end subroutine omp_set_dynamic
 
-          subroutine omp_set_nested(enable) bind(c)
+          subroutine omp_set_nested(nested) bind(c)
             use omp_lib_kinds
-            logical (kind=omp_logical_kind), value :: enable
+            logical (kind=omp_logical_kind), value :: nested
           end subroutine omp_set_nested
 
           function omp_get_num_threads() bind(c)
@@ -166,16 +166,16 @@
             integer (kind=omp_integer_kind), value :: level
           end function omp_get_team_size
 
-          subroutine omp_set_schedule(kind, modifier) bind(c)
+          subroutine omp_set_schedule(kind, chunk_size) bind(c)
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
-            integer (kind=omp_integer_kind), value :: modifier
+            integer (kind=omp_integer_kind), value :: chunk_size
           end subroutine omp_set_schedule
 
-          subroutine omp_get_schedule(kind, modifier) bind(c)
+          subroutine omp_get_schedule(kind, chunk_size) bind(c)
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
-            integer (kind=omp_integer_kind) modifier
+            integer (kind=omp_integer_kind) chunk_size
           end subroutine omp_get_schedule
 
           function omp_get_proc_bind() bind(c)
@@ -183,6 +183,38 @@
             integer (kind=omp_proc_bind_kind) omp_get_proc_bind
           end function omp_get_proc_bind
 
+          function omp_get_num_places() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_num_places
+          end function omp_get_num_places
+
+          function omp_get_place_num_procs(place_num) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind), value :: place_num
+            integer (kind=omp_integer_kind) omp_get_place_num_procs
+          end function omp_get_place_num_procs
+
+          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind), value :: place_num
+            integer (kind=kmp_pointer_kind) ids(*)
+          end subroutine omp_get_place_proc_ids
+
+          function omp_get_place_num() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_place_num
+          end function omp_get_place_num
+
+          function omp_get_partition_num_places() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_partition_num_places
+          end function omp_get_partition_num_places
+
+          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) place_nums(*)
+          end subroutine omp_get_partition_place_nums
+
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
             real (kind=kmp_double_kind) omp_get_wtime
@@ -198,9 +230,9 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(dflt_device) bind(c)
+          subroutine omp_set_default_device(device_num) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_integer_kind), value :: dflt_device
+            integer (kind=omp_integer_kind), value :: device_num
           end subroutine omp_set_default_device
 
           function omp_get_num_devices() bind(c)
@@ -228,88 +260,98 @@
             logical (kind=omp_logical_kind) omp_is_initial_device
           end function omp_is_initial_device
 
-          subroutine omp_init_lock(lockvar) bind(c)
+          function omp_get_initial_device() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_initial_device
+          end function omp_get_initial_device
+
+          subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_init_lock
 
-          subroutine omp_destroy_lock(lockvar) bind(c)
+          subroutine omp_destroy_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_destroy_lock
 
-          subroutine omp_set_lock(lockvar) bind(c)
+          subroutine omp_set_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_set_lock
 
-          subroutine omp_unset_lock(lockvar) bind(c)
+          subroutine omp_unset_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end subroutine omp_unset_lock
 
-          function omp_test_lock(lockvar) bind(c)
+          function omp_test_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             logical (kind=omp_logical_kind) omp_test_lock
-            integer (kind=omp_lock_kind) lockvar
+            integer (kind=omp_lock_kind) svar
           end function omp_test_lock
 
-          subroutine omp_init_nest_lock(lockvar) bind(c)
+          subroutine omp_init_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_init_nest_lock
 
-          subroutine omp_destroy_nest_lock(lockvar) bind(c)
+          subroutine omp_destroy_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_destroy_nest_lock
 
-          subroutine omp_set_nest_lock(lockvar) bind(c)
+          subroutine omp_set_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_set_nest_lock
 
-          subroutine omp_unset_nest_lock(lockvar) bind(c)
+          subroutine omp_unset_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end subroutine omp_unset_nest_lock
 
-          function omp_test_nest_lock(lockvar) bind(c)
+          function omp_test_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_test_nest_lock
-            integer (kind=omp_nest_lock_kind) lockvar
+            integer (kind=omp_nest_lock_kind) nvar
           end function omp_test_nest_lock
 
+          function omp_get_max_task_priority() bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind) omp_get_max_task_priority
+          end function omp_get_max_task_priority
+
 !         ***
 !         *** kmp_* entry points
 !         ***
@@ -368,6 +410,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
+          subroutine kmp_set_disp_num_buffers(num) bind(c)
+            use omp_lib_kinds
+            integer (kind=omp_integer_kind), value :: num
+          end subroutine kmp_set_disp_num_buffers
+
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) kmp_set_affinity
@@ -422,6 +469,13 @@
             integer (kind=kmp_size_t_kind), value :: size
           end function kmp_malloc
 
+          function kmp_aligned_malloc(size, alignment) bind(c)
+            use omp_lib_kinds
+            integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+            integer (kind=kmp_size_t_kind), value :: size
+            integer (kind=kmp_size_t_kind), value :: alignment
+          end function kmp_aligned_malloc
+
           function kmp_calloc(nelem, elsize) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_pointer_kind) kmp_calloc
@@ -453,16 +507,16 @@
             logical (kind=omp_logical_kind) kmp_get_cancellation_status
           end function kmp_get_cancellation_status
 
-          subroutine omp_init_lock_with_hint(lockvar, lockhint) bind(c)
+          subroutine omp_init_lock_with_hint(svar, hint) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
-            integer (kind=omp_lock_hint_kind), value :: lockhint
+            integer (kind=omp_lock_kind) svar
+            integer (kind=omp_lock_hint_kind), value :: hint
           end subroutine omp_init_lock_with_hint
 
-          subroutine omp_init_nest_lock_with_hint(lockvar, lockhint) bind(c)
+          subroutine omp_init_nest_lock_with_hint(nvar, hint) bind(c)
             use omp_lib_kinds
-            integer (kind=omp_lock_kind) lockvar
-            integer (kind=omp_lock_hint_kind), value :: lockhint
+            integer (kind=omp_nest_lock_kind) nvar
+            integer (kind=omp_lock_hint_kind), value :: hint
           end subroutine omp_init_nest_lock_with_hint
 
         end interface

runtime/src/include/45/omp_lib.h.var

@@ -1,4 +1,4 @@
-! include/41/omp_lib.h.var
+! include/45/omp_lib.h.var
 
 !
 !//===----------------------------------------------------------------------===//
@@ -63,19 +63,19 @@
 !       *** omp_* entry points
 !       ***
 
-        subroutine omp_set_num_threads(nthreads) bind(c)
+        subroutine omp_set_num_threads(num_threads) bind(c)
           import
-          integer (kind=omp_integer_kind), value :: nthreads
+          integer (kind=omp_integer_kind), value :: num_threads
         end subroutine omp_set_num_threads
 
-        subroutine omp_set_dynamic(enable) bind(c)
+        subroutine omp_set_dynamic(dynamic_threads) bind(c)
           import
-          logical (kind=omp_logical_kind), value :: enable
+          logical (kind=omp_logical_kind), value :: dynamic_threads
         end subroutine omp_set_dynamic
 
-        subroutine omp_set_nested(enable) bind(c)
+        subroutine omp_set_nested(nested) bind(c)
           import
-          logical (kind=omp_logical_kind), value :: enable
+          logical (kind=omp_logical_kind), value :: nested
         end subroutine omp_set_nested
 
         function omp_get_num_threads() bind(c)
@@ -155,16 +155,16 @@
           integer (kind=omp_integer_kind), value :: level
         end function omp_get_team_size
 
-        subroutine omp_set_schedule(kind, modifier) bind(c)
+        subroutine omp_set_schedule(kind, chunk_size) bind(c)
           import
           integer (kind=omp_sched_kind), value :: kind
-          integer (kind=omp_integer_kind), value :: modifier
+          integer (kind=omp_integer_kind), value :: chunk_size
         end subroutine omp_set_schedule
 
-        subroutine omp_get_schedule(kind, modifier) bind(c)
+        subroutine omp_get_schedule(kind, chunk_size) bind(c)
           import
           integer (kind=omp_sched_kind) kind
-          integer (kind=omp_integer_kind) modifier
+          integer (kind=omp_integer_kind) chunk_size
         end subroutine omp_get_schedule
 
         function omp_get_proc_bind() bind(c)
@@ -172,6 +172,38 @@
           integer (kind=omp_proc_bind_kind) omp_get_proc_bind
         end function omp_get_proc_bind
 
+        function omp_get_num_places() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_num_places
+        end function omp_get_num_places
+
+        function omp_get_place_num_procs(place_num) bind(c)
+          import
+          integer (kind=omp_integer_kind), value :: place_num
+          integer (kind=omp_integer_kind) omp_get_place_num_procs
+        end function omp_get_place_num_procs
+
+        subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          import
+          integer (kind=omp_integer_kind), value :: place_num
+          integer (kind=kmp_pointer_kind) ids(*)
+        end subroutine omp_get_place_proc_ids
+
+        function omp_get_place_num() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_place_num
+        end function omp_get_place_num
+
+        function omp_get_partition_num_places() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_partition_num_places
+        end function omp_get_partition_num_places
+
+        subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          import
+          integer (kind=kmp_pointer_kind) place_nums(*)
+        end subroutine omp_get_partition_place_nums
+
         function omp_get_wtime() bind(c)
           double precision omp_get_wtime
         end function omp_get_wtime
@@ -185,9 +217,9 @@
           integer (kind=omp_integer_kind) omp_get_default_device
         end function omp_get_default_device
 
-        subroutine omp_set_default_device(dflt_device) bind(c)
+        subroutine omp_set_default_device(device_num) bind(c)
           import
-          integer (kind=omp_integer_kind), value :: dflt_device
+          integer (kind=omp_integer_kind), value :: device_num
         end subroutine omp_set_default_device
 
         function omp_get_num_devices() bind(c)
@@ -210,88 +242,98 @@
           logical (kind=omp_logical_kind) omp_is_initial_device
         end function omp_is_initial_device
 
-        subroutine omp_init_lock(lockvar) bind(c)
+        function omp_get_initial_device() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_initial_device
+        end function omp_get_initial_device
+
+        subroutine omp_init_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_init_lock
 
-        subroutine omp_destroy_lock(lockvar) bind(c)
+        subroutine omp_destroy_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_destroy_lock
 
-        subroutine omp_set_lock(lockvar) bind(c)
+        subroutine omp_set_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_set_lock
 
-        subroutine omp_unset_lock(lockvar) bind(c)
+        subroutine omp_unset_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end subroutine omp_unset_lock
 
-        function omp_test_lock(lockvar) bind(c)
+        function omp_test_lock(svar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_lock
 !DIR$ ENDIF
           import
           logical (kind=omp_logical_kind) omp_test_lock
-          integer (kind=omp_lock_kind) lockvar
+          integer (kind=omp_lock_kind) svar
         end function omp_test_lock
 
-        subroutine omp_init_nest_lock(lockvar) bind(c)
+        subroutine omp_init_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_init_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_init_nest_lock
 
-        subroutine omp_destroy_nest_lock(lockvar) bind(c)
+        subroutine omp_destroy_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_destroy_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_destroy_nest_lock
 
-        subroutine omp_set_nest_lock(lockvar) bind(c)
+        subroutine omp_set_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_set_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_set_nest_lock
 
-        subroutine omp_unset_nest_lock(lockvar) bind(c)
+        subroutine omp_unset_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_unset_nest_lock
 !DIR$ ENDIF
           import
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end subroutine omp_unset_nest_lock
 
-        function omp_test_nest_lock(lockvar) bind(c)
+        function omp_test_nest_lock(nvar) bind(c)
 !DIR$ IF(__INTEL_COMPILER.GE.1400)
 !DIR$ attributes known_intrinsic :: omp_test_nest_lock
 !DIR$ ENDIF
           import
           integer (kind=omp_integer_kind) omp_test_nest_lock
-          integer (kind=omp_nest_lock_kind) lockvar
+          integer (kind=omp_nest_lock_kind) nvar
         end function omp_test_nest_lock
 
+        function omp_get_max_task_priority() bind(c)
+          import
+          integer (kind=omp_integer_kind) omp_get_max_task_priority
+        end function omp_get_max_task_priority
+
 !       ***
 !       *** kmp_* entry points
 !       ***
@@ -349,6 +391,11 @@
           integer (kind=omp_integer_kind) kmp_get_library
         end function kmp_get_library
 
+        subroutine kmp_set_disp_num_buffers(num) bind(c)
+          import
+          integer (kind=omp_integer_kind), value :: num
+        end subroutine kmp_set_disp_num_buffers
+
         function kmp_set_affinity(mask) bind(c)
           import
           integer (kind=omp_integer_kind) kmp_set_affinity
@@ -403,6 +450,13 @@
           integer (kind=kmp_size_t_kind), value :: size
         end function kmp_malloc
 
+        function kmp_aligned_malloc(size, alignment) bind(c)
+          import
+          integer (kind=kmp_pointer_kind) kmp_aligned_malloc
+          integer (kind=kmp_size_t_kind), value :: size
+          integer (kind=kmp_size_t_kind), value :: alignment
+        end function kmp_aligned_malloc
+
         function kmp_calloc(nelem, elsize) bind(c)
           import
           integer (kind=kmp_pointer_kind) kmp_calloc
@@ -428,16 +482,16 @@
         subroutine kmp_set_warnings_off() bind(c)
         end subroutine kmp_set_warnings_off
 
-        subroutine omp_init_lock_with_hint(lockvar, lockhint) bind(c)
+        subroutine omp_init_lock_with_hint(svar, hint) bind(c)
           import
-          integer (kind=omp_lock_kind) lockvar
-          integer (kind=omp_lock_hint_kind), value :: lockhint
+          integer (kind=omp_lock_kind) svar
+          integer (kind=omp_lock_hint_kind), value :: hint
         end subroutine omp_init_lock_with_hint
 
-        subroutine omp_init_nest_lock_with_hint(lockvar, lockhint) bind(c)
+        subroutine omp_init_nest_lock_with_hint(nvar, hint) bind(c)
           import
-          integer (kind=omp_lock_kind) lockvar
-          integer (kind=omp_lock_hint_kind), value :: lockhint
+          integer (kind=omp_nest_lock_kind) nvar
+          integer (kind=omp_lock_hint_kind), value :: hint
         end subroutine omp_init_nest_lock_with_hint
 
       end interface
@@ -482,6 +536,7 @@
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_set_nest_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_unset_nest_lock
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_test_nest_lock
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: omp_get_max_task_priority
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_stacksize
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_stacksize_s
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_blocktime
@@ -494,6 +549,7 @@
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_stacksize_s
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_blocktime
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_library
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_disp_num_buffers
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_set_affinity
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity_max_proc
@@ -503,6 +559,7 @@
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_unset_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_get_affinity_mask_proc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_malloc
+!DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_aligned_malloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_calloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_realloc
 !DIR$ ATTRIBUTES OFFLOAD:MIC :: kmp_free
@@ -551,6 +608,7 @@
 !$omp declare target(omp_set_nest_lock )
 !$omp declare target(omp_unset_nest_lock )
 !$omp declare target(omp_test_nest_lock )
+!$omp declare target(omp_get_max_task_priority )
 !$omp declare target(kmp_set_stacksize )
 !$omp declare target(kmp_set_stacksize_s )
 !$omp declare target(kmp_set_blocktime )
@@ -563,6 +621,7 @@
 !$omp declare target(kmp_get_stacksize_s )
 !$omp declare target(kmp_get_blocktime )
 !$omp declare target(kmp_get_library )
+!$omp declare target(kmp_set_disp_num_buffers )
 !$omp declare target(kmp_set_affinity )
 !$omp declare target(kmp_get_affinity )
 !$omp declare target(kmp_get_affinity_max_proc )
@@ -572,6 +631,7 @@
 !$omp declare target(kmp_unset_affinity_mask_proc )
 !$omp declare target(kmp_get_affinity_mask_proc )
 !$omp declare target(kmp_malloc )
+!$omp declare target(kmp_aligned_malloc )
 !$omp declare target(kmp_calloc )
 !$omp declare target(kmp_realloc )
 !$omp declare target(kmp_free )

runtime/src/include/45/ompt.h.var

@@ -1,5 +1,5 @@
 /*
- * include/41/ompt.h.var
+ * include/45/ompt.h.var
  */
 
 #ifndef __OMPT__
@@ -171,7 +171,10 @@
     macro (ompt_event_destroy_lock,             ompt_wait_callback_t,          59) /* lock destruction */       \
     macro (ompt_event_destroy_nest_lock,        ompt_wait_callback_t,          60) /* nest lock destruction */  \
                                                                                                                 \
-    macro (ompt_event_flush,                    ompt_callback_t,               61) /* after executing flush */
+    macro (ompt_event_flush,                    ompt_callback_t,               61) /* after executing flush */  \
+                                                                                                                \
+    macro (ompt_event_task_dependences,         ompt_task_dependences_callback_t, 69) /* report task dependences  */\
+    macro (ompt_event_task_dependence_pair,     ompt_task_pair_callback_t,     70) /* report task dependence pair */
 
 
 
@@ -206,6 +209,23 @@ typedef struct ompt_frame_s {
 } ompt_frame_t;
 
 
+/*---------------------
+ * dependences types
+ *---------------------*/
+
+typedef enum ompt_task_dependence_flag_e {
+    // a two bit field for the dependence type
+    ompt_task_dependence_type_out   = 1,
+    ompt_task_dependence_type_in    = 2,
+    ompt_task_dependence_type_inout = 3,
+} ompt_task_dependence_flag_t;
+
+typedef struct ompt_task_dependence_s {
+    void *variable_addr;
+    uint32_t  dependence_flags;
+} ompt_task_dependence_t;
+
+
 /*****************************************************************************
  * enumerations for thread states and runtime events
  *****************************************************************************/
@@ -299,13 +319,13 @@ typedef void (*ompt_new_parallel_callback_t) (
     ompt_parallel_id_t parallel_id,   /* id of parallel region        */
     uint32_t requested_team_size,     /* number of threads in team    */
     void *parallel_function,          /* pointer to outlined function */
-    ompt_invoker_t invoker            /* who invokes master task?    */ 
+    ompt_invoker_t invoker            /* who invokes master task?     */
 );
 
 typedef void (*ompt_end_parallel_callback_t) (
     ompt_parallel_id_t parallel_id,   /* id of parallel region       */
     ompt_task_id_t task_id,           /* id of task                  */
-    ompt_invoker_t invoker            /* who invokes master task?    */ 
+    ompt_invoker_t invoker            /* who invokes master task?    */
 );
 
 /* tasks */
@@ -325,6 +345,13 @@ typedef void (*ompt_new_task_callback_t) (
     void *task_function               /* pointer to outlined function */
 );
 
+/* task dependences */
+typedef void (*ompt_task_dependences_callback_t) (
+    ompt_task_id_t task_id,            /* ID of task with dependences */
+    const ompt_task_dependence_t *deps,/* vector of task dependences  */
+    int ndeps                          /* number of dependences       */
+);
+
 /* program */
 typedef void (*ompt_control_callback_t) (
     uint64_t command,                 /* command of control call      */
@@ -461,14 +488,15 @@ OMPT_API_FUNCTION(int, ompt_get_callback, (
  ***************************************************************************/
 
 /* control */
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+// FIXME: remove workaround for clang
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp declare target
 #endif
 void ompt_control(
     uint64_t command,
     uint64_t modifier
 );
-#if defined(_OPENMP) && (_OPENMP >= 201307)
+#if !defined(__clang__) && defined(_OPENMP) && (_OPENMP >= 201307)
 #pragma omp end declare target
 #endif
 

runtime/src/kmp_affinity.h

@@ -15,7 +15,490 @@
 #ifndef KMP_AFFINITY_H
 #define KMP_AFFINITY_H
 
-extern int __kmp_affinity_compact; /* Affinity 'compact' value */
+#include "kmp_os.h"
+#include "kmp.h"
+
+#if KMP_AFFINITY_SUPPORTED
+#if KMP_USE_HWLOC
+class KMPHwlocAffinity: public KMPAffinity {
+public:
+    class Mask : public KMPAffinity::Mask {
+        hwloc_cpuset_t mask;
+    public:
+        Mask() { mask = hwloc_bitmap_alloc(); this->zero(); }
+        ~Mask() { hwloc_bitmap_free(mask); }
+        void set(int i) override { hwloc_bitmap_set(mask, i); }
+        bool is_set(int i) const override { return hwloc_bitmap_isset(mask, i); }
+        void clear(int i) override { hwloc_bitmap_clr(mask, i); }
+        void zero() override { hwloc_bitmap_zero(mask); }
+        void copy(const KMPAffinity::Mask* src) override {
+            const Mask* convert = static_cast<const Mask*>(src);
+            hwloc_bitmap_copy(mask, convert->mask);
+        }
+        void bitwise_and(const KMPAffinity::Mask* rhs) override {
+            const Mask* convert = static_cast<const Mask*>(rhs);
+            hwloc_bitmap_and(mask, mask, convert->mask);
+        }
+        void bitwise_or(const KMPAffinity::Mask * rhs) override {
+            const Mask* convert = static_cast<const Mask*>(rhs);
+            hwloc_bitmap_or(mask, mask, convert->mask);
+        }
+        void bitwise_not() override { hwloc_bitmap_not(mask, mask); }
+        int begin() const override { return hwloc_bitmap_first(mask); }
+        int end() const override { return -1; }
+        int next(int previous) const override { return hwloc_bitmap_next(mask, previous); }
+        int get_system_affinity(bool abort_on_error) override {
+            KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+              "Illegal get affinity operation when not capable");
+            int retval = hwloc_get_cpubind(__kmp_hwloc_topology, mask, HWLOC_CPUBIND_THREAD);
+            if (retval >= 0) {
+                return 0;
+            }
+            int error = errno;
+            if (abort_on_error) {
+                __kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
+            }
+            return error;
+        }
+        int set_system_affinity(bool abort_on_error) const override {
+            KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+              "Illegal get affinity operation when not capable");
+            int retval = hwloc_set_cpubind(__kmp_hwloc_topology, mask, HWLOC_CPUBIND_THREAD);
+            if (retval >= 0) {
+                return 0;
+            }
+            int error = errno;
+            if (abort_on_error) {
+                __kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
+            }
+            return error;
+        }
+        int get_proc_group() const override {
+            int i;
+            int group = -1;
+# if KMP_OS_WINDOWS
+            if (__kmp_num_proc_groups == 1) {
+                return 1;
+            }
+            for (i = 0; i < __kmp_num_proc_groups; i++) {
+                // On windows, the long type is always 32 bits
+                unsigned long first_32_bits = hwloc_bitmap_to_ith_ulong(mask, i*2);
+                unsigned long second_32_bits = hwloc_bitmap_to_ith_ulong(mask, i*2+1);
+                if (first_32_bits == 0 && second_32_bits == 0) {
+                    continue;
+                }
+                if (group >= 0) {
+                    return -1;
+                }
+                group = i;
+            }
+# endif /* KMP_OS_WINDOWS */
+            return group;
+        }
+    };
+    void determine_capable(const char* var) override {
+        const hwloc_topology_support* topology_support;
+        if(__kmp_hwloc_topology == NULL) {
+            if(hwloc_topology_init(&__kmp_hwloc_topology) < 0) {
+                __kmp_hwloc_error = TRUE;
+                if(__kmp_affinity_verbose)
+                    KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_init()");
+            }
+            if(hwloc_topology_load(__kmp_hwloc_topology) < 0) {
+                __kmp_hwloc_error = TRUE;
+                if(__kmp_affinity_verbose)
+                    KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_load()");
+            }
+        }
+        topology_support = hwloc_topology_get_support(__kmp_hwloc_topology);
+        // Is the system capable of setting/getting this thread's affinity?
+        // also, is topology discovery possible? (pu indicates ability to discover processing units)
+        // and finally, were there no errors when calling any hwloc_* API functions?
+        if(topology_support && topology_support->cpubind->set_thisthread_cpubind &&
+           topology_support->cpubind->get_thisthread_cpubind &&
+           topology_support->discovery->pu &&
+           !__kmp_hwloc_error)
+        {
+            // enables affinity according to KMP_AFFINITY_CAPABLE() macro
+            KMP_AFFINITY_ENABLE(TRUE);
+        } else {
+            // indicate that hwloc didn't work and disable affinity
+            __kmp_hwloc_error = TRUE;
+            KMP_AFFINITY_DISABLE();
+        }
+    }
+    void bind_thread(int which) override {
+        KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+          "Illegal set affinity operation when not capable");
+        KMPAffinity::Mask *mask;
+        KMP_CPU_ALLOC_ON_STACK(mask);
+        KMP_CPU_ZERO(mask);
+        KMP_CPU_SET(which, mask);
+        __kmp_set_system_affinity(mask, TRUE);
+        KMP_CPU_FREE_FROM_STACK(mask);
+    }
+    KMPAffinity::Mask* allocate_mask() override { return new Mask();  }
+    void deallocate_mask(KMPAffinity::Mask* m) override { delete m; }
+    KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
+    void deallocate_mask_array(KMPAffinity::Mask* array) override {
+        Mask* hwloc_array = static_cast<Mask*>(array);
+        delete[] hwloc_array;
+    }
+    KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
+        Mask* hwloc_array = static_cast<Mask*>(array);
+        return &(hwloc_array[index]);
+    }
+    api_type get_api_type() const override { return HWLOC; }
+};
+#endif /* KMP_USE_HWLOC */
+
+#if KMP_OS_LINUX
+/*
+ * On some of the older OS's that we build on, these constants aren't present
+ * in <asm/unistd.h> #included from <sys.syscall.h>.  They must be the same on
+ * all systems of the same arch where they are defined, and they cannot change.
+ * stone forever.
+ */
+#include <sys/syscall.h>
+# if KMP_ARCH_X86 || KMP_ARCH_ARM
+#  ifndef __NR_sched_setaffinity
+#   define __NR_sched_setaffinity  241
+#  elif __NR_sched_setaffinity != 241
+#   error Wrong code for setaffinity system call.
+#  endif /* __NR_sched_setaffinity */
+#  ifndef __NR_sched_getaffinity
+#   define __NR_sched_getaffinity  242
+#  elif __NR_sched_getaffinity != 242
+#   error Wrong code for getaffinity system call.
+#  endif /* __NR_sched_getaffinity */
+# elif KMP_ARCH_AARCH64
+#  ifndef __NR_sched_setaffinity
+#   define __NR_sched_setaffinity  122
+#  elif __NR_sched_setaffinity != 122
+#   error Wrong code for setaffinity system call.
+#  endif /* __NR_sched_setaffinity */
+#  ifndef __NR_sched_getaffinity
+#   define __NR_sched_getaffinity  123
+#  elif __NR_sched_getaffinity != 123
+#   error Wrong code for getaffinity system call.
+#  endif /* __NR_sched_getaffinity */
+# elif KMP_ARCH_X86_64
+#  ifndef __NR_sched_setaffinity
+#   define __NR_sched_setaffinity  203
+#  elif __NR_sched_setaffinity != 203
+#   error Wrong code for setaffinity system call.
+#  endif /* __NR_sched_setaffinity */
+#  ifndef __NR_sched_getaffinity
+#   define __NR_sched_getaffinity  204
+#  elif __NR_sched_getaffinity != 204
+#   error Wrong code for getaffinity system call.
+#  endif /* __NR_sched_getaffinity */
+# elif KMP_ARCH_PPC64
+#  ifndef __NR_sched_setaffinity
+#   define __NR_sched_setaffinity  222
+#  elif __NR_sched_setaffinity != 222
+#   error Wrong code for setaffinity system call.
+#  endif /* __NR_sched_setaffinity */
+#  ifndef __NR_sched_getaffinity
+#   define __NR_sched_getaffinity  223
+#  elif __NR_sched_getaffinity != 223
+#   error Wrong code for getaffinity system call.
+#  endif /* __NR_sched_getaffinity */
+#  elif KMP_ARCH_MIPS
+#   ifndef __NR_sched_setaffinity
+#    define __NR_sched_setaffinity  4239
+#   elif __NR_sched_setaffinity != 4239
+#    error Wrong code for setaffinity system call.
+#   endif /* __NR_sched_setaffinity */
+#   ifndef __NR_sched_getaffinity
+#    define __NR_sched_getaffinity  4240
+#   elif __NR_sched_getaffinity != 4240
+#    error Wrong code for getaffinity system call.
+#   endif /* __NR_sched_getaffinity */
+#  elif KMP_ARCH_MIPS64
+#   ifndef __NR_sched_setaffinity
+#    define __NR_sched_setaffinity  5195
+#   elif __NR_sched_setaffinity != 5195
+#    error Wrong code for setaffinity system call.
+#   endif /* __NR_sched_setaffinity */
+#   ifndef __NR_sched_getaffinity
+#    define __NR_sched_getaffinity  5196
+#   elif __NR_sched_getaffinity != 5196
+#    error Wrong code for getaffinity system call.
+#   endif /* __NR_sched_getaffinity */
+#  error Unknown or unsupported architecture
+# endif /* KMP_ARCH_* */
+class KMPNativeAffinity : public KMPAffinity {
+    class Mask : public KMPAffinity::Mask {
+        typedef unsigned char mask_t;
+        static const int BITS_PER_MASK_T = sizeof(mask_t)*CHAR_BIT;
+    public:
+        mask_t* mask;
+        Mask() { mask = (mask_t*)__kmp_allocate(__kmp_affin_mask_size); }
+        ~Mask() { if (mask) __kmp_free(mask); }
+        void set(int i) override { mask[i/BITS_PER_MASK_T] |= ((mask_t)1 << (i % BITS_PER_MASK_T)); }
+        bool is_set(int i) const override { return (mask[i/BITS_PER_MASK_T] & ((mask_t)1 << (i % BITS_PER_MASK_T))); }
+        void clear(int i) override { mask[i/BITS_PER_MASK_T] &= ~((mask_t)1 << (i % BITS_PER_MASK_T)); }
+        void zero() override {
+            for (size_t i=0; i<__kmp_affin_mask_size; ++i)
+                mask[i] = 0;
+        }
+        void copy(const KMPAffinity::Mask* src) override {
+            const Mask * convert = static_cast<const Mask*>(src);
+            for (size_t i=0; i<__kmp_affin_mask_size; ++i)
+                mask[i] = convert->mask[i];
+        }
+        void bitwise_and(const KMPAffinity::Mask* rhs) override {
+            const Mask * convert = static_cast<const Mask*>(rhs);
+            for (size_t i=0; i<__kmp_affin_mask_size; ++i)
+                mask[i] &= convert->mask[i];
+        }
+        void bitwise_or(const KMPAffinity::Mask* rhs) override {
+            const Mask * convert = static_cast<const Mask*>(rhs);
+            for (size_t i=0; i<__kmp_affin_mask_size; ++i)
+                mask[i] |= convert->mask[i];
+        }
+        void bitwise_not() override {
+            for (size_t i=0; i<__kmp_affin_mask_size; ++i)
+                mask[i] = ~(mask[i]);
+        }
+        int begin() const override {
+            int retval = 0;
+            while (retval < end() && !is_set(retval))
+                ++retval;
+            return retval;
+        }
+        int end() const override { return __kmp_affin_mask_size*BITS_PER_MASK_T; }
+        int next(int previous) const override {
+            int retval = previous+1;
+            while (retval < end() && !is_set(retval))
+                ++retval;
+            return retval;
+        }
+        int get_system_affinity(bool abort_on_error) override {
+            KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+              "Illegal get affinity operation when not capable");
+            int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
+            if (retval >= 0) {
+                return 0;
+            }
+            int error = errno;
+            if (abort_on_error) {
+                __kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
+            }
+            return error;
+        }
+        int set_system_affinity(bool abort_on_error) const override {
+            KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+              "Illegal get affinity operation when not capable");
+            int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
+            if (retval >= 0) {
+                return 0;
+            }
+            int error = errno;
+            if (abort_on_error) {
+                __kmp_msg(kmp_ms_fatal, KMP_MSG( FatalSysError ), KMP_ERR( error ), __kmp_msg_null);
+            }
+            return error;
+        }
+    };
+    void determine_capable(const char* env_var) override {
+        __kmp_affinity_determine_capable(env_var);
+    }
+    void bind_thread(int which) override {
+        __kmp_affinity_bind_thread(which);
+    }
+    KMPAffinity::Mask* allocate_mask() override {
+        KMPNativeAffinity::Mask* retval = new Mask();
+        return retval;
+    }
+    void deallocate_mask(KMPAffinity::Mask* m) override {
+        KMPNativeAffinity::Mask* native_mask = static_cast<KMPNativeAffinity::Mask*>(m);
+        delete m;
+    }
+    KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
+    void deallocate_mask_array(KMPAffinity::Mask* array) override {
+        Mask* linux_array = static_cast<Mask*>(array);
+        delete[] linux_array;
+    }
+    KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
+        Mask* linux_array = static_cast<Mask*>(array);
+        return &(linux_array[index]);
+    }
+    api_type get_api_type() const override { return NATIVE_OS; }
+};
+#endif /* KMP_OS_LINUX */
+
+#if KMP_OS_WINDOWS
+class KMPNativeAffinity : public KMPAffinity {
+    class Mask : public KMPAffinity::Mask {
+        typedef ULONG_PTR mask_t;
+        static const int BITS_PER_MASK_T = sizeof(mask_t)*CHAR_BIT;
+        mask_t* mask;
+    public:
+        Mask() { mask = (mask_t*)__kmp_allocate(sizeof(mask_t)*__kmp_num_proc_groups); }
+        ~Mask() { if (mask) __kmp_free(mask); }
+        void set(int i) override { mask[i/BITS_PER_MASK_T] |= ((mask_t)1 << (i % BITS_PER_MASK_T)); }
+        bool is_set(int i) const override { return (mask[i/BITS_PER_MASK_T] & ((mask_t)1 << (i % BITS_PER_MASK_T))); }
+        void clear(int i) override { mask[i/BITS_PER_MASK_T] &= ~((mask_t)1 << (i % BITS_PER_MASK_T)); }
+        void zero() override {
+            for (size_t i=0; i<__kmp_num_proc_groups; ++i)
+                mask[i] = 0;
+        }
+        void copy(const KMPAffinity::Mask* src) override {
+            const Mask * convert = static_cast<const Mask*>(src);
+            for (size_t i=0; i<__kmp_num_proc_groups; ++i)
+                mask[i] = convert->mask[i];
+        }
+        void bitwise_and(const KMPAffinity::Mask* rhs) override {
+            const Mask * convert = static_cast<const Mask*>(rhs);
+            for (size_t i=0; i<__kmp_num_proc_groups; ++i)
+                mask[i] &= convert->mask[i];
+        }
+        void bitwise_or(const KMPAffinity::Mask* rhs) override {
+            const Mask * convert = static_cast<const Mask*>(rhs);
+            for (size_t i=0; i<__kmp_num_proc_groups; ++i)
+                mask[i] |= convert->mask[i];
+        }
+        void bitwise_not() override {
+            for (size_t i=0; i<__kmp_num_proc_groups; ++i)
+                mask[i] = ~(mask[i]);
+        }
+        int begin() const override {
+            int retval = 0;
+            while (retval < end() && !is_set(retval))
+                ++retval;
+            return retval;
+        }
+        int end() const override { return __kmp_num_proc_groups*BITS_PER_MASK_T; }
+        int next(int previous) const override {
+            int retval = previous+1;
+            while (retval < end() && !is_set(retval))
+                ++retval;
+            return retval;
+        }
+        int set_system_affinity(bool abort_on_error) const override {
+            if (__kmp_num_proc_groups > 1) {
+                // Check for a valid mask.
+                GROUP_AFFINITY ga;
+                int group = get_proc_group();
+                if (group < 0) {
+                    if (abort_on_error) {
+                        KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
+                    }
+                    return -1;
+                }
+                // Transform the bit vector into a GROUP_AFFINITY struct
+                // and make the system call to set affinity.
+                ga.Group = group;
+                ga.Mask = mask[group];
+                ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
+
+                KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
+                if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetThreadAffMask ),
+                                  KMP_ERR( error ), __kmp_msg_null);
+                    }
+                    return error;
+                }
+            } else {
+                if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetThreadAffMask ),
+                                  KMP_ERR( error ), __kmp_msg_null);
+                    }
+                    return error;
+                }
+            }
+            return 0;
+        }
+        int get_system_affinity(bool abort_on_error) override {
+            if (__kmp_num_proc_groups > 1) {
+                this->zero();
+                GROUP_AFFINITY ga;
+                KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
+                if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
+                                  KMP_ERR(error), __kmp_msg_null);
+                    }
+                    return error;
+                }
+                if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups) || (ga.Mask == 0)) {
+                    return -1;
+                }
+                mask[ga.Group] = ga.Mask;
+            } else {
+                mask_t newMask, sysMask, retval;
+                if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
+                                  KMP_ERR(error), __kmp_msg_null);
+                    }
+                    return error;
+                }
+                retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
+                if (! retval) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
+                                  KMP_ERR(error), __kmp_msg_null);
+                    }
+                    return error;
+                }
+                newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
+                if (! newMask) {
+                    DWORD error = GetLastError();
+                    if (abort_on_error) {
+                        __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
+                                  KMP_ERR(error), __kmp_msg_null);
+                    }
+                }
+                *mask = retval;
+            }
+            return 0;
+        }
+        int get_proc_group() const override {
+            int group = -1;
+            if (__kmp_num_proc_groups == 1) {
+                return 1;
+            }
+            for (int i = 0; i < __kmp_num_proc_groups; i++) {
+                if (mask[i] == 0)
+                    continue;
+                if (group >= 0)
+                    return -1;
+                group = i;
+            }
+            return group;
+        }
+    };
+    void determine_capable(const char* env_var) override {
+        __kmp_affinity_determine_capable(env_var);
+    }
+    void bind_thread(int which) override {
+        __kmp_affinity_bind_thread(which);
+    }
+    KMPAffinity::Mask* allocate_mask() override { return new Mask();  }
+    void deallocate_mask(KMPAffinity::Mask* m) override { delete m; }
+    KMPAffinity::Mask* allocate_mask_array(int num) override { return new Mask[num]; }
+    void deallocate_mask_array(KMPAffinity::Mask* array) override {
+        Mask* windows_array = static_cast<Mask*>(array);
+        delete[] windows_array;
+    }
+    KMPAffinity::Mask* index_mask_array(KMPAffinity::Mask* array, int index) override {
+        Mask* windows_array = static_cast<Mask*>(array);
+        return &(windows_array[index]);
+    }
+    api_type get_api_type() const override { return NATIVE_OS; }
+};
+#endif /* KMP_OS_WINDOWS */
+#endif /* KMP_AFFINITY_SUPPORTED */
 
 class Address {
 public:
@@ -112,32 +595,6 @@ __kmp_affinity_cmp_Address_labels(const void *a, const void *b)
 }
 
 
-static int
-__kmp_affinity_cmp_Address_child_num(const void *a, const void *b)
-{
-    const Address *aa = (const Address *)&(((AddrUnsPair *)a)
-      ->first);
-    const Address *bb = (const Address *)&(((AddrUnsPair *)b)
-      ->first);
-    unsigned depth = aa->depth;
-    unsigned i;
-    KMP_DEBUG_ASSERT(depth == bb->depth);
-    KMP_DEBUG_ASSERT((unsigned)__kmp_affinity_compact <= depth);
-    KMP_DEBUG_ASSERT(__kmp_affinity_compact >= 0);
-    for (i = 0; i < (unsigned)__kmp_affinity_compact; i++) {
-        int j = depth - i - 1;
-        if (aa->childNums[j] < bb->childNums[j]) return -1;
-        if (aa->childNums[j] > bb->childNums[j]) return 1;
-    }
-    for (; i < depth; i++) {
-        int j = i - __kmp_affinity_compact;
-        if (aa->childNums[j] < bb->childNums[j]) return -1;
-        if (aa->childNums[j] > bb->childNums[j]) return 1;
-    }
-    return 0;
-}
-
-
 /** A structure for holding machine-specific hierarchy info to be computed once at init.
     This structure represents a mapping of threads to the actual machine hierarchy, or to
     our best guess at what the hierarchy might be, for the purpose of performing an

runtime/src/kmp_alloc.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_alloc.c -- private/shared dyanmic memory allocation and management
+ * kmp_alloc.cpp -- private/shared dynamic memory allocation and management
  */
 
 
@@ -1427,8 +1427,37 @@ void *
 kmpc_malloc( size_t size )
 {
     void * ptr;
-        ptr = bget( __kmp_entry_thread(), (bufsize) size );
+    ptr = bget( __kmp_entry_thread(), (bufsize)(size + sizeof(ptr)) );
+    if( ptr != NULL ) {
+        // save allocated pointer just before one returned to user
+        *(void**)ptr = ptr;
+        ptr = (void**)ptr + 1;
+    }
+    return ptr;
+}
 
+#define IS_POWER_OF_TWO(n) (((n)&((n)-1))==0)
+
+void *
+kmpc_aligned_malloc( size_t size, size_t alignment )
+{
+    void * ptr;
+    void * ptr_allocated;
+    KMP_DEBUG_ASSERT( alignment < 32 * 1024 ); // Alignment should not be too big
+    if( !IS_POWER_OF_TWO(alignment) ) {
+        // AC: do we need to issue a warning here?
+        errno = EINVAL;
+        return NULL;
+    }
+    size = size + sizeof( void* ) + alignment;
+    ptr_allocated = bget( __kmp_entry_thread(), (bufsize)size );
+    if( ptr_allocated != NULL ) {
+        // save allocated pointer just before one returned to user
+        ptr = (void*)(((kmp_uintptr_t)ptr_allocated + sizeof( void* ) + alignment) & ~(alignment - 1));
+        *((void**)ptr - 1) = ptr_allocated;
+    } else {
+        ptr = NULL;
+    }
     return ptr;
 }
 
@@ -1436,8 +1465,12 @@ void *
 kmpc_calloc( size_t nelem, size_t elsize )
 {
     void * ptr;
-        ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize) );
-
+    ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize + sizeof(ptr)) );
+    if( ptr != NULL ) {
+        // save allocated pointer just before one returned to user
+        *(void**)ptr = ptr;
+        ptr = (void**)ptr + 1;
+    }
     return ptr;
 }
 
@@ -1445,19 +1478,27 @@ void *
 kmpc_realloc( void * ptr, size_t size )
 {
     void * result = NULL;
-
-        if ( ptr == NULL ) {
-            // If pointer is NULL, realloc behaves like malloc.
-            result = bget( __kmp_entry_thread(), (bufsize) size );
-        } else if ( size == 0 ) {
-            // If size is 0, realloc behaves like free.
-            // The thread must be registered by the call to kmpc_malloc() or kmpc_calloc() before.
-            // So it should be safe to call __kmp_get_thread(), not __kmp_entry_thread().
-            brel( __kmp_get_thread(), ptr );
-        } else {
-            result = bgetr( __kmp_entry_thread(), ptr, (bufsize) size );
-        }; // if
-
+    if ( ptr == NULL ) {
+        // If pointer is NULL, realloc behaves like malloc.
+        result = bget( __kmp_entry_thread(), (bufsize)(size + sizeof(ptr)) );
+        // save allocated pointer just before one returned to user
+        if( result != NULL ) {
+            *(void**)result = result;
+            result = (void**)result + 1;
+        }
+    } else if ( size == 0 ) {
+        // If size is 0, realloc behaves like free.
+        // The thread must be registered by the call to kmpc_malloc() or kmpc_calloc() before.
+        // So it should be safe to call __kmp_get_thread(), not __kmp_entry_thread().
+        KMP_ASSERT(*((void**)ptr - 1));
+        brel( __kmp_get_thread(), *((void**)ptr - 1) );
+    } else {
+        result = bgetr( __kmp_entry_thread(), *((void**)ptr - 1), (bufsize)(size + sizeof(ptr)) );
+        if( result != NULL ) {
+            *(void**)result = result;
+            result = (void**)result + 1;
+        }
+    }; // if
     return result;
 }
 
@@ -1470,9 +1511,11 @@ kmpc_free( void * ptr )
         return;
     }; // if
     if ( ptr != NULL ) {
-            kmp_info_t *th = __kmp_get_thread();
-            __kmp_bget_dequeue( th );         /* Release any queued buffers */
-            brel( th, ptr );
+        kmp_info_t *th = __kmp_get_thread();
+        __kmp_bget_dequeue( th );         /* Release any queued buffers */
+        // extract allocated pointer and free it
+        KMP_ASSERT(*((void**)ptr - 1));
+        brel( th, *((void**)ptr - 1) );
     };
 }
 
@@ -1489,7 +1532,7 @@ ___kmp_thread_malloc( kmp_info_t *th, size_t size KMP_SRC_LOC_DECL )
         (int) size
         KMP_SRC_LOC_PARM
     ) );
-        ptr = bget( th, (bufsize) size );
+    ptr = bget( th, (bufsize) size );
     KE_TRACE( 30, ( "<- __kmp_thread_malloc() returns %p\n", ptr ) );
     return ptr;
 }
@@ -1505,7 +1548,7 @@ ___kmp_thread_calloc( kmp_info_t *th, size_t nelem, size_t elsize KMP_SRC_LOC_DE
         (int) elsize
         KMP_SRC_LOC_PARM
     ) );
-        ptr = bgetz( th, (bufsize) (nelem * elsize) );
+    ptr = bgetz( th, (bufsize) (nelem * elsize) );
     KE_TRACE( 30, ( "<- __kmp_thread_calloc() returns %p\n", ptr ) );
     return ptr;
 }
@@ -1520,7 +1563,7 @@ ___kmp_thread_realloc( kmp_info_t *th, void *ptr, size_t size KMP_SRC_LOC_DECL )
         (int) size
         KMP_SRC_LOC_PARM
     ) );
-        ptr = bgetr( th, ptr, (bufsize) size );
+    ptr = bgetr( th, ptr, (bufsize) size );
     KE_TRACE( 30, ( "<- __kmp_thread_realloc() returns %p\n", ptr ) );
     return ptr;
 }
@@ -1535,8 +1578,8 @@ ___kmp_thread_free( kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL )
         KMP_SRC_LOC_PARM
     ) );
     if ( ptr != NULL ) {
-            __kmp_bget_dequeue( th );         /* Release any queued buffers */
-            brel( th, ptr );
+        __kmp_bget_dequeue( th );         /* Release any queued buffers */
+        brel( th, ptr );
     }
     KE_TRACE( 30, ( "<- __kmp_thread_free()\n" ) );
 }
@@ -1602,11 +1645,11 @@ ___kmp_allocate_align( size_t size, size_t alignment KMP_SRC_LOC_DECL )
     descr.size_aligned = size;
     descr.size_allocated = descr.size_aligned + sizeof( kmp_mem_descr_t ) + alignment;
 
-    #if KMP_DEBUG
-        descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
-    #else
+#if KMP_DEBUG
+    descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
+#else
     descr.ptr_allocated = malloc_src_loc( descr.size_allocated KMP_SRC_LOC_PARM );
-    #endif
+#endif
     KE_TRACE( 10, (
         "   malloc( %d ) returned %p\n",
         (int) descr.size_allocated,
@@ -1638,11 +1681,10 @@ ___kmp_allocate_align( size_t size, size_t alignment KMP_SRC_LOC_DECL )
     KMP_DEBUG_ASSERT( addr_descr + sizeof( kmp_mem_descr_t ) == addr_aligned );
     KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
     KMP_DEBUG_ASSERT( addr_aligned % alignment == 0 );
-
-    #ifdef KMP_DEBUG
-        memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
-            // Fill allocated memory block with 0xEF.
-    #endif
+#ifdef KMP_DEBUG
+    memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
+        // Fill allocated memory block with 0xEF.
+#endif
     memset( descr.ptr_aligned, 0x00, descr.size_aligned );
         // Fill the aligned memory block (which is intended for using by caller) with 0x00. Do not
         // put this filling under KMP_DEBUG condition! Many callers expect zeroed memory. (Padding
@@ -1653,7 +1695,6 @@ ___kmp_allocate_align( size_t size, size_t alignment KMP_SRC_LOC_DECL )
 
     KE_TRACE( 25, ( "<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned ) );
     return descr.ptr_aligned;
-
 } // func ___kmp_allocate_align
 
 
@@ -1666,13 +1707,11 @@ ___kmp_allocate_align( size_t size, size_t alignment KMP_SRC_LOC_DECL )
 void *
 ___kmp_allocate( size_t size KMP_SRC_LOC_DECL )
 {
-
     void * ptr;
     KE_TRACE( 25, ( "-> __kmp_allocate( %d ) called from %s:%d\n", (int) size KMP_SRC_LOC_PARM ) );
-        ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
+    ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
     KE_TRACE( 25, ( "<- __kmp_allocate() returns %p\n", ptr ) );
     return ptr;
-
 } // func ___kmp_allocate
 
 #if (BUILD_MEMORY==FIRST_TOUCH)
@@ -1680,13 +1719,8 @@ void *
 __kmp_ft_page_allocate(size_t size)
 {
   void *adr, *aadr;
-#if KMP_OS_LINUX
-  /* TODO: Use this function to get page size everywhere */
-  int page_size = getpagesize();
-#else
-  /* TODO: Find windows function to get page size and use it everywhere */
-  int page_size = PAGE_SIZE;
-#endif /* KMP_OS_LINUX */
+
+  const int page_size = KMP_GET_PAGE_SIZE();
 
   adr = (void *) __kmp_thread_malloc( __kmp_get_thread(),
                                     size + page_size + KMP_PTR_SKIP);
@@ -1726,7 +1760,7 @@ ___kmp_page_allocate( size_t size KMP_SRC_LOC_DECL )
         (int) size
         KMP_SRC_LOC_PARM
     ) );
-        ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
+    ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
     KE_TRACE( 25, ( "<- __kmp_page_allocate( %d ) returns %p\n", (int) size, ptr ) );
     return ptr;
 } // ___kmp_page_allocate
@@ -1738,49 +1772,45 @@ ___kmp_page_allocate( size_t size KMP_SRC_LOC_DECL )
 void
 ___kmp_free( void * ptr KMP_SRC_LOC_DECL )
 {
+    kmp_mem_descr_t descr;
+    kmp_uintptr_t   addr_allocated;        // Address returned by malloc().
+    kmp_uintptr_t   addr_aligned;          // Aligned address passed by caller.
 
-        kmp_mem_descr_t descr;
-        kmp_uintptr_t   addr_allocated;        // Address returned by malloc().
-        kmp_uintptr_t   addr_aligned;          // Aligned address passed by caller.
+    KE_TRACE( 25, ( "-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
+    KMP_ASSERT( ptr != NULL );
 
-        KE_TRACE( 25, ( "-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
-        KMP_ASSERT( ptr != NULL );
+    descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr - sizeof( kmp_mem_descr_t ) );
 
-        descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr - sizeof( kmp_mem_descr_t ) );
+    KE_TRACE( 26, ( "   __kmp_free:     "
+                    "ptr_allocated=%p, size_allocated=%d, "
+                    "ptr_aligned=%p, size_aligned=%d\n",
+                    descr.ptr_allocated, (int) descr.size_allocated,
+                    descr.ptr_aligned, (int) descr.size_aligned ));
 
-        KE_TRACE( 26, ( "   __kmp_free:     "
-                        "ptr_allocated=%p, size_allocated=%d, "
-                        "ptr_aligned=%p, size_aligned=%d\n",
-                        descr.ptr_allocated, (int) descr.size_allocated,
-                        descr.ptr_aligned, (int) descr.size_aligned ));
+    addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
+    addr_aligned   = (kmp_uintptr_t) descr.ptr_aligned;
 
-        addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
-        addr_aligned   = (kmp_uintptr_t) descr.ptr_aligned;
+    KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
+    KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
+    KMP_DEBUG_ASSERT( addr_allocated + sizeof( kmp_mem_descr_t ) <= addr_aligned );
+    KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
+    KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
 
-        KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
-        KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
-        KMP_DEBUG_ASSERT( addr_allocated + sizeof( kmp_mem_descr_t ) <= addr_aligned );
-        KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
-        KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
-
-        #ifdef KMP_DEBUG
-            memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
-                // Fill memory block with 0xEF, it helps catch using freed memory.
-        #endif
-
-        #ifndef LEAK_MEMORY
-            KE_TRACE( 10, ( "   free( %p )\n", descr.ptr_allocated ) );
-        # ifdef KMP_DEBUG
-            _free_src_loc( descr.ptr_allocated, _file_, _line_ );
-        # else
-            free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
-        # endif
-        #endif
+    #ifdef KMP_DEBUG
+        memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
+            // Fill memory block with 0xEF, it helps catch using freed memory.
+    #endif
 
+    #ifndef LEAK_MEMORY
+        KE_TRACE( 10, ( "   free( %p )\n", descr.ptr_allocated ) );
+    # ifdef KMP_DEBUG
+        _free_src_loc( descr.ptr_allocated, _file_, _line_ );
+    # else
+        free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
+    # endif
+    #endif
     KMP_MB();
-
     KE_TRACE( 25, ( "<- __kmp_free() returns\n" ) );
-
 } // func ___kmp_free
 
 /* ------------------------------------------------------------------------ */

runtime/src/kmp_atomic.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_atomic.c -- ATOMIC implementation routines
+ * kmp_atomic.cpp -- ATOMIC implementation routines
  */
 
 
@@ -184,6 +184,9 @@ There are versions here for integers of size 1,2,4 and 8 bytes both signed and u
     __kmpc_atomic_fixed1_wr
     __kmpc_atomic_fixed1_xor
     __kmpc_atomic_fixed1_xor_cpt
+    __kmpc_atomic_fixed1u_add_fp
+    __kmpc_atomic_fixed1u_sub_fp
+    __kmpc_atomic_fixed1u_mul_fp
     __kmpc_atomic_fixed1u_div
     __kmpc_atomic_fixed1u_div_cpt
     __kmpc_atomic_fixed1u_div_cpt_rev
@@ -240,6 +243,9 @@ There are versions here for integers of size 1,2,4 and 8 bytes both signed and u
     __kmpc_atomic_fixed2_wr
     __kmpc_atomic_fixed2_xor
     __kmpc_atomic_fixed2_xor_cpt
+    __kmpc_atomic_fixed2u_add_fp
+    __kmpc_atomic_fixed2u_sub_fp
+    __kmpc_atomic_fixed2u_mul_fp
     __kmpc_atomic_fixed2u_div
     __kmpc_atomic_fixed2u_div_cpt
     __kmpc_atomic_fixed2u_div_cpt_rev
@@ -296,6 +302,9 @@ There are versions here for integers of size 1,2,4 and 8 bytes both signed and u
     __kmpc_atomic_fixed4_wr
     __kmpc_atomic_fixed4_xor
     __kmpc_atomic_fixed4_xor_cpt
+    __kmpc_atomic_fixed4u_add_fp
+    __kmpc_atomic_fixed4u_sub_fp
+    __kmpc_atomic_fixed4u_mul_fp
     __kmpc_atomic_fixed4u_div
     __kmpc_atomic_fixed4u_div_cpt
     __kmpc_atomic_fixed4u_div_cpt_rev
@@ -352,6 +361,9 @@ There are versions here for integers of size 1,2,4 and 8 bytes both signed and u
     __kmpc_atomic_fixed8_wr
     __kmpc_atomic_fixed8_xor
     __kmpc_atomic_fixed8_xor_cpt
+    __kmpc_atomic_fixed8u_add_fp
+    __kmpc_atomic_fixed8u_sub_fp
+    __kmpc_atomic_fixed8u_mul_fp
     __kmpc_atomic_fixed8u_div
     __kmpc_atomic_fixed8u_div_cpt
     __kmpc_atomic_fixed8u_div_cpt_rev
@@ -1377,6 +1389,21 @@ ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE)
 }
 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
 
+// -------------------------------------------------------------------------
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+// -------------------------------------------------------------------------
+#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
+ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE)                                         \
+    OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG)                                                       \
+    OP_CMPXCHG_REV(TYPE,BITS,OP)                                                                \
+}
+#define ATOMIC_CRITICAL_REV_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG)           \
+ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE)                                         \
+    OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG)                                        \
+    OP_CRITICAL_REV(OP,LCK_ID)                                                \
+}
+#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
+
 // RHS=float8
 ATOMIC_CMPXCHG_MIX( fixed1, char,       mul,  8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_float8
 ATOMIC_CMPXCHG_MIX( fixed1, char,       div,  8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_float8
@@ -1394,26 +1421,38 @@ ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, K
 // RHS=float16 (deprecated, to be removed when we are sure the compiler does not use them)
 #if KMP_HAVE_QUAD
 ATOMIC_CMPXCHG_MIX( fixed1,  char,       add,  8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add_fp
+ATOMIC_CMPXCHG_MIX( fixed1u, uchar,      add,  8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_add_fp
 ATOMIC_CMPXCHG_MIX( fixed1,  char,       sub,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_fp
+ATOMIC_CMPXCHG_MIX( fixed1u, uchar,      sub,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_fp
 ATOMIC_CMPXCHG_MIX( fixed1,  char,       mul,  8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_fp
+ATOMIC_CMPXCHG_MIX( fixed1u, uchar,      mul,  8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_mul_fp
 ATOMIC_CMPXCHG_MIX( fixed1,  char,       div,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_fp
 ATOMIC_CMPXCHG_MIX( fixed1u, uchar,      div,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_fp
 
 ATOMIC_CMPXCHG_MIX( fixed2,  short,      add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add_fp
+ATOMIC_CMPXCHG_MIX( fixed2u, ushort,     add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_add_fp
 ATOMIC_CMPXCHG_MIX( fixed2,  short,      sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_fp
+ATOMIC_CMPXCHG_MIX( fixed2u, ushort,     sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_fp
 ATOMIC_CMPXCHG_MIX( fixed2,  short,      mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_fp
+ATOMIC_CMPXCHG_MIX( fixed2u, ushort,     mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_mul_fp
 ATOMIC_CMPXCHG_MIX( fixed2,  short,      div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_fp
 ATOMIC_CMPXCHG_MIX( fixed2u, ushort,     div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_fp
 
 ATOMIC_CMPXCHG_MIX( fixed4,  kmp_int32,  add, 32, +, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_add_fp
+ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_add_fp
 ATOMIC_CMPXCHG_MIX( fixed4,  kmp_int32,  sub, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_sub_fp
+ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_sub_fp
 ATOMIC_CMPXCHG_MIX( fixed4,  kmp_int32,  mul, 32, *, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_mul_fp
+ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_mul_fp
 ATOMIC_CMPXCHG_MIX( fixed4,  kmp_int32,  div, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_div_fp
 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_div_fp
 
 ATOMIC_CMPXCHG_MIX( fixed8,  kmp_int64,  add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add_fp
+ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_add_fp
 ATOMIC_CMPXCHG_MIX( fixed8,  kmp_int64,  sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_fp
+ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_fp
 ATOMIC_CMPXCHG_MIX( fixed8,  kmp_int64,  mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_fp
+ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_mul_fp
 ATOMIC_CMPXCHG_MIX( fixed8,  kmp_int64,  div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_fp
 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_fp
 
@@ -1431,6 +1470,39 @@ ATOMIC_CRITICAL_FP( float10, long double,    add, +, fp, _Quad, 10r,   1 )
 ATOMIC_CRITICAL_FP( float10, long double,    sub, -, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_sub_fp
 ATOMIC_CRITICAL_FP( float10, long double,    mul, *, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_mul_fp
 ATOMIC_CRITICAL_FP( float10, long double,    div, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_fp
+
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+// Reverse operations
+ATOMIC_CMPXCHG_REV_MIX( fixed1,  char,       sub_rev,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar,      sub_rev,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed1,  char,       div_rev,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar,      div_rev,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_rev_fp
+
+ATOMIC_CMPXCHG_REV_MIX( fixed2,  short,      sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort,     sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed2,  short,      div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort,     div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_rev_fp
+
+ATOMIC_CMPXCHG_REV_MIX( fixed4,  kmp_int32,  sub_rev, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed4,  kmp_int32,  div_rev, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_div_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_div_rev_fp
+
+ATOMIC_CMPXCHG_REV_MIX( fixed8,  kmp_int64,  sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed8,  kmp_int64,  div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_rev_fp
+
+ATOMIC_CMPXCHG_REV_MIX( float4,  kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( float4,  kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_rev_fp
+
+ATOMIC_CMPXCHG_REV_MIX( float8,  kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_rev_fp
+ATOMIC_CMPXCHG_REV_MIX( float8,  kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_rev_fp
+
+ATOMIC_CRITICAL_REV_FP( float10, long double,    sub_rev, -, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_sub_rev_fp
+ATOMIC_CRITICAL_REV_FP( float10, long double,    div_rev, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_rev_fp
+#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
+
 #endif
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
@@ -1900,6 +1972,92 @@ ATOMIC_CMPXCHG_CPT( float8,  div_cpt, kmp_real64, 64, /,  KMP_ARCH_X86 )  // __k
 ATOMIC_CMPXCHG_CPT( float8,  mul_cpt, kmp_real64, 64, *,  KMP_ARCH_X86 )  // __kmpc_atomic_float8_mul_cpt
 //              TYPE_ID,OP_ID, TYPE,          OP,  GOMP_FLAG
 
+//////////////////////////////////
+
+// CAPTURE routines for mixed types RHS=float16
+#if KMP_HAVE_QUAD
+
+// Beginning of a definition (provides name, parameters, gebug trace)
+//     TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
+//     OP_ID   - operation identifier (add, sub, mul, ...)
+//     TYPE    - operands' type
+#define ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE)       \
+TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs, int flag ) \
+{                                                                                         \
+    KMP_DEBUG_ASSERT( __kmp_init_serial );                                                \
+    KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
+
+// -------------------------------------------------------------------------
+#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG)       \
+ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE)                    \
+    TYPE new_value;                                                        \
+    OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG)                                     \
+    OP_CMPXCHG_CPT(TYPE,BITS,OP)                                           \
+}
+
+// -------------------------------------------------------------------------
+#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG)         \
+ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE)                    \
+    TYPE new_value;                                                        \
+    OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG)  /* send assignment */                              \
+    OP_CRITICAL_CPT(OP##=,LCK_ID)  /* send assignment */                                      \
+}
+
+ATOMIC_CMPXCHG_CPT_MIX( fixed1,  char,       add_cpt,  8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar,      add_cpt,  8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1,  char,       sub_cpt,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar,      sub_cpt,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1,  char,       mul_cpt,  8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar,      mul_cpt,  8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1,  char,       div_cpt,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar,      div_cpt,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt_fp
+
+ATOMIC_CMPXCHG_CPT_MIX( fixed2,  short,      add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort,     add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2,  short,      sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort,     sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2,  short,      mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort,     mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2,  short,      div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort,     div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt_fp
+
+ATOMIC_CMPXCHG_CPT_MIX( fixed4,  kmp_int32,  add_cpt, 32, +, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4,  kmp_int32,  sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4,  kmp_int32,  mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4,  kmp_int32,  div_cpt, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_div_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_div_cpt_fp
+
+ATOMIC_CMPXCHG_CPT_MIX( fixed8,  kmp_int64,  add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8,  kmp_int64,  sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8,  kmp_int64,  mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8,  kmp_int64,  div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt_fp
+
+ATOMIC_CMPXCHG_CPT_MIX( float4,  kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float4,  kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float4,  kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float4,  kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt_fp
+
+ATOMIC_CMPXCHG_CPT_MIX( float8,  kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_add_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float8,  kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float8,  kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_mul_cpt_fp
+ATOMIC_CMPXCHG_CPT_MIX( float8,  kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt_fp
+
+ATOMIC_CRITICAL_CPT_MIX( float10, long double, add_cpt, +, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_add_cpt_fp
+ATOMIC_CRITICAL_CPT_MIX( float10, long double, sub_cpt, -, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_sub_cpt_fp
+ATOMIC_CRITICAL_CPT_MIX( float10, long double, mul_cpt, *, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_mul_cpt_fp
+ATOMIC_CRITICAL_CPT_MIX( float10, long double, div_cpt, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_cpt_fp
+
+#endif //KMP_HAVE_QUAD
+
+///////////////////////////////////
+
 // ------------------------------------------------------------------------
 // Routines for C/C++ Reduction operators && and ||
 // ------------------------------------------------------------------------
@@ -2380,6 +2538,61 @@ ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,   1 )
 #endif
 #endif
 
+// Capture reverse for mixed type: RHS=float16
+#if KMP_HAVE_QUAD
+
+// Beginning of a definition (provides name, parameters, gebug trace)
+//     TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
+//     OP_ID   - operation identifier (add, sub, mul, ...)
+//     TYPE    - operands' type
+// -------------------------------------------------------------------------
+#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG)       \
+ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE)                    \
+    TYPE new_value;                                                        \
+    OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG)                                 \
+    OP_CMPXCHG_CPT_REV(TYPE,BITS,OP)                                       \
+}
+
+// -------------------------------------------------------------------------
+#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG)         \
+ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE)                    \
+    TYPE new_value;                                                        \
+    OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG)  /* send assignment */                              \
+    OP_CRITICAL_CPT_REV(OP,LCK_ID)  /* send assignment */                                      \
+}
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1,  char,       sub_cpt_rev,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar,      sub_cpt_rev,  8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1,  char,       div_cpt_rev,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar,      div_cpt_rev,  8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt_rev_fp
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2,  short,      sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort,     sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2,  short,      div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort,     div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt_rev_fp
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4,  kmp_int32,  sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4,  kmp_int32,  div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4_div_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 )            // __kmpc_atomic_fixed4u_div_cpt_rev_fp
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8,  kmp_int64,  sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8,  kmp_int64,  div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt_rev_fp
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( float4,  kmp_real32, sub_cpt_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( float4,  kmp_real32, div_cpt_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt_rev_fp
+
+ATOMIC_CMPXCHG_CPT_REV_MIX( float8,  kmp_real64, sub_cpt_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt_rev_fp
+ATOMIC_CMPXCHG_CPT_REV_MIX( float8,  kmp_real64, div_cpt_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt_rev_fp
+
+ATOMIC_CRITICAL_CPT_REV_MIX( float10, long double, sub_cpt_rev, -, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_sub_cpt_rev_fp
+ATOMIC_CRITICAL_CPT_REV_MIX( float10, long double, div_cpt_rev, /, fp, _Quad, 10r,   1 )            // __kmpc_atomic_float10_div_cpt_rev_fp
+
+#endif //KMP_HAVE_QUAD
+
+
 //   OpenMP 4.0 Capture-write (swap): {v = x; x = expr;}
 
 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE)                                                    \

runtime/src/kmp_atomic.h

@@ -35,6 +35,13 @@
 #if defined( __cplusplus ) && ( KMP_OS_WINDOWS )
     // create shortcuts for c99 complex types
 
+    // Visual Studio cannot have function parameters that have the
+    // align __declspec attribute, so we must remove it. (Compiler Error C2719)
+    #if KMP_COMPILER_MSVC
+    # undef KMP_DO_ALIGN
+    # define KMP_DO_ALIGN(alignment) /* Nothing */
+    #endif
+
     #if (_MSC_VER < 1600) && defined(_DEBUG)
         // Workaround for the problem of _DebugHeapTag unresolved external.
         // This problem prevented to use our static debug library for C tests
@@ -211,7 +218,7 @@
 
 // Compiler 12.0 changed alignment of 16 and 32-byte arguments (like _Quad
 // and kmp_cmplx128) on IA-32 architecture. The following aligned structures
-// are implemented to support the old alignment in 10.1, 11.0, 11.1 and 
+// are implemented to support the old alignment in 10.1, 11.0, 11.1 and
 // introduce the new alignment in 12.0. See CQ88405.
 #if KMP_ARCH_X86 && KMP_HAVE_QUAD
 
@@ -219,7 +226,7 @@
 
     #pragma pack( push, 4 )
 
-    
+
     struct KMP_DO_ALIGN( 4 ) Quad_a4_t {
         _Quad q;
 
@@ -371,7 +378,7 @@ static inline void
 __kmp_acquire_atomic_lock( kmp_atomic_lock_t *lck, kmp_int32 gtid )
 {
 #if OMPT_SUPPORT && OMPT_TRACE
-    if (ompt_enabled && 
+    if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_wait_atomic)) {
         ompt_callbacks.ompt_callback(ompt_event_wait_atomic)(
             (ompt_wait_id_t) lck);
@@ -381,7 +388,7 @@ __kmp_acquire_atomic_lock( kmp_atomic_lock_t *lck, kmp_int32 gtid )
     __kmp_acquire_queuing_lock( lck, gtid );
 
 #if OMPT_SUPPORT && OMPT_TRACE
-    if (ompt_enabled && 
+    if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_acquired_atomic)) {
         ompt_callbacks.ompt_callback(ompt_event_acquired_atomic)(
             (ompt_wait_id_t) lck);
@@ -670,26 +677,38 @@ void __kmpc_atomic_float4_div_float8( ident_t *id_ref, int gtid, kmp_real32 * lh
 // RHS=float16 (deprecated, to be removed when we are sure the compiler does not use them)
 #if KMP_HAVE_QUAD
 void __kmpc_atomic_fixed1_add_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed1u_add_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs );
 void __kmpc_atomic_fixed1_sub_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed1u_sub_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs );
 void __kmpc_atomic_fixed1_mul_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed1u_mul_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs );
 void __kmpc_atomic_fixed1_div_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
 void __kmpc_atomic_fixed1u_div_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs );
 
 void __kmpc_atomic_fixed2_add_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2u_add_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs );
 void __kmpc_atomic_fixed2_sub_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2u_sub_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs );
 void __kmpc_atomic_fixed2_mul_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2u_mul_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs );
 void __kmpc_atomic_fixed2_div_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
 void __kmpc_atomic_fixed2u_div_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs );
 
 void __kmpc_atomic_fixed4_add_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4u_add_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed4_sub_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4u_sub_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed4_mul_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4u_mul_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed4_div_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed4u_div_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs );
 
 void __kmpc_atomic_fixed8_add_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8u_add_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed8_sub_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8u_sub_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed8_mul_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8u_mul_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed8_div_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
 void __kmpc_atomic_fixed8u_div_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs );
 
@@ -707,6 +726,27 @@ void __kmpc_atomic_float10_add_fp( ident_t *id_ref, int gtid, long double * lhs,
 void __kmpc_atomic_float10_sub_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs );
 void __kmpc_atomic_float10_mul_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs );
 void __kmpc_atomic_float10_div_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs );
+
+// Reverse operations
+void __kmpc_atomic_fixed1_sub_rev_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed1_div_rev_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed1u_div_rev_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2_sub_rev_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2_div_rev_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed2u_div_rev_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4_sub_rev_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4_div_rev_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed4u_div_rev_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8_sub_rev_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8_div_rev_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs );
+void __kmpc_atomic_fixed8u_div_rev_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs );
+void __kmpc_atomic_float4_sub_rev_fp(  ident_t *id_ref, int gtid, float * lhs, _Quad rhs );
+void __kmpc_atomic_float4_div_rev_fp(  ident_t *id_ref, int gtid, float * lhs, _Quad rhs );
+void __kmpc_atomic_float8_sub_rev_fp(  ident_t *id_ref, int gtid, double * lhs, _Quad rhs );
+void __kmpc_atomic_float8_div_rev_fp(  ident_t *id_ref, int gtid, double * lhs, _Quad rhs );
+void __kmpc_atomic_float10_sub_rev_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs );
+void __kmpc_atomic_float10_div_rev_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs );
+
 #endif // KMP_HAVE_QUAD
 
 // RHS=cmplx8
@@ -1016,10 +1056,89 @@ kmp_cmplx80	__kmpc_atomic_cmplx10_swp( ident_t *id_ref, int gtid, kmp_cmplx80 *
 CPLX128_LEG 	__kmpc_atomic_cmplx16_swp( ident_t *id_ref, int gtid, CPLX128_LEG * lhs, CPLX128_LEG rhs );
 #if ( KMP_ARCH_X86 )
     Quad_a16_t		__kmpc_atomic_float16_a16_swp( ident_t *id_ref, int gtid, Quad_a16_t * lhs, Quad_a16_t rhs );
-    kmp_cmplx128_a16_t 	__kmpc_atomic_cmplx16_a16_swp( ident_t *id_ref, int gtid, kmp_cmplx128_a16_t * lhs, kmp_cmplx128_a16_t rhs );
+    kmp_cmplx128_a16_t  __kmpc_atomic_cmplx16_a16_swp( ident_t *id_ref, int gtid, kmp_cmplx128_a16_t * lhs, kmp_cmplx128_a16_t rhs );
 #endif
 #endif
 
+// Capture routines for mixed types (RHS=float16)
+#if KMP_HAVE_QUAD
+
+char __kmpc_atomic_fixed1_add_cpt_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+char __kmpc_atomic_fixed1_sub_cpt_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+char __kmpc_atomic_fixed1_mul_cpt_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+char __kmpc_atomic_fixed1_div_cpt_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+unsigned char  __kmpc_atomic_fixed1u_add_cpt_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+unsigned char __kmpc_atomic_fixed1u_sub_cpt_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+unsigned char __kmpc_atomic_fixed1u_mul_cpt_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+unsigned char __kmpc_atomic_fixed1u_div_cpt_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+
+short __kmpc_atomic_fixed2_add_cpt_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+short __kmpc_atomic_fixed2_sub_cpt_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+short __kmpc_atomic_fixed2_mul_cpt_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+short __kmpc_atomic_fixed2_div_cpt_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+unsigned short __kmpc_atomic_fixed2u_add_cpt_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+unsigned short __kmpc_atomic_fixed2u_sub_cpt_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+unsigned short __kmpc_atomic_fixed2u_mul_cpt_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+unsigned short __kmpc_atomic_fixed2u_div_cpt_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+
+kmp_int32 __kmpc_atomic_fixed4_add_cpt_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs, int flag );
+kmp_int32 __kmpc_atomic_fixed4_sub_cpt_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs, int flag );
+kmp_int32 __kmpc_atomic_fixed4_mul_cpt_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs, int flag );
+kmp_int32 __kmpc_atomic_fixed4_div_cpt_fp(  ident_t *id_ref, int gtid, kmp_int32 * lhs, _Quad rhs, int flag );
+kmp_uint32 __kmpc_atomic_fixed4u_add_cpt_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+kmp_uint32 __kmpc_atomic_fixed4u_sub_cpt_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+kmp_uint32 __kmpc_atomic_fixed4u_mul_cpt_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+kmp_uint32 __kmpc_atomic_fixed4u_div_cpt_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+
+kmp_int64 __kmpc_atomic_fixed8_add_cpt_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs, int flag );
+kmp_int64 __kmpc_atomic_fixed8_sub_cpt_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs, int flag );
+kmp_int64 __kmpc_atomic_fixed8_mul_cpt_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs, int flag );
+kmp_int64 __kmpc_atomic_fixed8_div_cpt_fp(  ident_t *id_ref, int gtid, kmp_int64 * lhs, _Quad rhs, int flag );
+kmp_uint64 __kmpc_atomic_fixed8u_add_cpt_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+kmp_uint64 __kmpc_atomic_fixed8u_sub_cpt_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+kmp_uint64 __kmpc_atomic_fixed8u_mul_cpt_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+kmp_uint64 __kmpc_atomic_fixed8u_div_cpt_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+
+float __kmpc_atomic_float4_add_cpt_fp(  ident_t *id_ref, int gtid, kmp_real32 * lhs, _Quad rhs, int flag );
+float __kmpc_atomic_float4_sub_cpt_fp(  ident_t *id_ref, int gtid, kmp_real32 * lhs, _Quad rhs, int flag );
+float __kmpc_atomic_float4_mul_cpt_fp(  ident_t *id_ref, int gtid, kmp_real32 * lhs, _Quad rhs, int flag );
+float __kmpc_atomic_float4_div_cpt_fp(  ident_t *id_ref, int gtid, kmp_real32 * lhs, _Quad rhs, int flag );
+
+double __kmpc_atomic_float8_add_cpt_fp(  ident_t *id_ref, int gtid, kmp_real64 * lhs, _Quad rhs, int flag );
+double __kmpc_atomic_float8_sub_cpt_fp(  ident_t *id_ref, int gtid, kmp_real64 * lhs, _Quad rhs, int flag );
+double __kmpc_atomic_float8_mul_cpt_fp(  ident_t *id_ref, int gtid, kmp_real64 * lhs, _Quad rhs, int flag );
+double __kmpc_atomic_float8_div_cpt_fp(  ident_t *id_ref, int gtid, kmp_real64 * lhs, _Quad rhs, int flag );
+
+long double __kmpc_atomic_float10_add_cpt_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+long double __kmpc_atomic_float10_sub_cpt_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+long double __kmpc_atomic_float10_mul_cpt_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+long double __kmpc_atomic_float10_div_cpt_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+
+char            __kmpc_atomic_fixed1_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+unsigned char   __kmpc_atomic_fixed1u_sub_cpt_rev_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+char            __kmpc_atomic_fixed1_div_cpt_rev_fp(  ident_t *id_ref, int gtid, char * lhs, _Quad rhs, int flag );
+unsigned char   __kmpc_atomic_fixed1u_div_cpt_rev_fp( ident_t *id_ref, int gtid, unsigned char * lhs, _Quad rhs, int flag );
+short           __kmpc_atomic_fixed2_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+unsigned short  __kmpc_atomic_fixed2u_sub_cpt_rev_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+short           __kmpc_atomic_fixed2_div_cpt_rev_fp(  ident_t *id_ref, int gtid, short * lhs, _Quad rhs, int flag );
+unsigned short  __kmpc_atomic_fixed2u_div_cpt_rev_fp( ident_t *id_ref, int gtid, unsigned short * lhs, _Quad rhs, int flag );
+kmp_int32       __kmpc_atomic_fixed4_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, kmp_int32  * lhs, _Quad  rhs, int flag );
+kmp_uint32      __kmpc_atomic_fixed4u_sub_cpt_rev_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+kmp_int32       __kmpc_atomic_fixed4_div_cpt_rev_fp(  ident_t *id_ref, int gtid, kmp_int32  * lhs, _Quad  rhs, int flag );
+kmp_uint32      __kmpc_atomic_fixed4u_div_cpt_rev_fp( ident_t *id_ref, int gtid, kmp_uint32 * lhs, _Quad rhs, int flag );
+kmp_int64       __kmpc_atomic_fixed8_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, kmp_int64  * lhs, _Quad  rhs, int flag );
+kmp_uint64      __kmpc_atomic_fixed8u_sub_cpt_rev_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+kmp_int64       __kmpc_atomic_fixed8_div_cpt_rev_fp(  ident_t *id_ref, int gtid, kmp_int64  * lhs, _Quad  rhs, int flag );
+kmp_uint64      __kmpc_atomic_fixed8u_div_cpt_rev_fp( ident_t *id_ref, int gtid, kmp_uint64 * lhs, _Quad rhs, int flag );
+float           __kmpc_atomic_float4_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, float * lhs, _Quad rhs, int flag );
+float           __kmpc_atomic_float4_div_cpt_rev_fp(  ident_t *id_ref, int gtid, float * lhs, _Quad rhs, int flag );
+double          __kmpc_atomic_float8_sub_cpt_rev_fp(  ident_t *id_ref, int gtid, double * lhs, _Quad rhs, int flag );
+double          __kmpc_atomic_float8_div_cpt_rev_fp(  ident_t *id_ref, int gtid, double * lhs, _Quad rhs, int flag );
+long double     __kmpc_atomic_float10_sub_cpt_rev_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+long double     __kmpc_atomic_float10_div_cpt_rev_fp( ident_t *id_ref, int gtid, long double * lhs, _Quad rhs, int flag );
+
+#endif // KMP_HAVE_QUAD
+
 // End of OpenMP 4.0 capture
 
 #endif //OMP_40_ENABLED

runtime/src/kmp_barrier.cpp

@@ -25,6 +25,8 @@
 #define USE_NGO_STORES 1
 #endif // KMP_MIC
 
+#include "tsan_annotations.h"
+
 #if KMP_MIC && USE_NGO_STORES
 // ICV copying
 #define ngo_load(src)            __m512d Vt = _mm512_load_pd((void *)(src))
@@ -48,7 +50,7 @@ __kmp_linear_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid
                             void (*reduce)(void *, void *)
                             USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_linear_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
@@ -107,8 +109,11 @@ __kmp_linear_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid
             if (reduce) {
                 KA_TRACE(100, ("__kmp_linear_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n", gtid,
                                team->t.t_id, tid, __kmp_gtid_from_tid(i, team), team->t.t_id, i));
+                ANNOTATE_REDUCE_AFTER(reduce);
                 (*reduce)(this_thr->th.th_local.reduce_data,
                           other_threads[i]->th.th_local.reduce_data);
+                ANNOTATE_REDUCE_BEFORE(reduce);
+                ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
             }
         }
         // Don't have to worry about sleep bit here or atomic since team setting
@@ -125,7 +130,7 @@ __kmp_linear_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gti
                              int propagate_icvs
                              USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_linear_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_linear_release);
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_team_t *team;
 
@@ -144,7 +149,7 @@ __kmp_linear_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gti
         if (nproc > 1) {
 #if KMP_BARRIER_ICV_PUSH
             {
-                KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+                KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
                 if (propagate_icvs) {
                     ngo_load(&team->t.t_implicit_task_taskdata[0].td_icvs);
                     for (i=1; i<nproc; ++i) {
@@ -220,7 +225,7 @@ __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                           void (*reduce)(void *, void *)
                           USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_tree_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
@@ -274,7 +279,10 @@ __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                 KA_TRACE(100, ("__kmp_tree_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                team->t.t_id, child_tid));
+                ANNOTATE_REDUCE_AFTER(reduce);
                 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
+                ANNOTATE_REDUCE_BEFORE(reduce);
+                ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
             }
             child++;
             child_tid++;
@@ -315,7 +323,7 @@ __kmp_tree_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                            int propagate_icvs
                            USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_tree_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_tree_release);
     register kmp_team_t *team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc;
@@ -385,7 +393,7 @@ __kmp_tree_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
 
 #if KMP_BARRIER_ICV_PUSH
             {
-                KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+                KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
                 if (propagate_icvs) {
                     __kmp_init_implicit_task(team->t.t_ident, team->t.t_threads[child_tid],
                                              team, child_tid, FALSE);
@@ -418,7 +426,7 @@ __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                            void (*reduce)(void *, void *)
                            USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hyper_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_info_t **other_threads = team->t.t_threads;
@@ -498,7 +506,10 @@ __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                 KA_TRACE(100, ("__kmp_hyper_barrier_gather: T#%d(%d:%d) += T#%d(%d:%u)\n",
                                gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                team->t.t_id, child_tid));
+                ANNOTATE_REDUCE_AFTER(reduce);
                 (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
+                ANNOTATE_REDUCE_BEFORE(reduce);
+                ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
             }
         }
     }
@@ -524,7 +535,7 @@ __kmp_hyper_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, int gtid
                             int propagate_icvs
                             USE_ITT_BUILD_ARG(void *itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hyper_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hyper_release);
     register kmp_team_t    *team;
     register kmp_bstate_t  *thr_bar       = & this_thr -> th.th_bar[ bt ].bb;
     register kmp_info_t   **other_threads;
@@ -731,7 +742,7 @@ __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr,
                                   int gtid, int tid, void (*reduce) (void *, void *)
                                   USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hier_gather);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_gather);
     register kmp_team_t *team = this_thr->th.th_team;
     register kmp_bstate_t *thr_bar = & this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc = this_thr->th.th_team_nproc;
@@ -772,12 +783,15 @@ __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr,
                 flag.wait(this_thr, FALSE
                           USE_ITT_BUILD_ARG(itt_sync_obj) );
                 if (reduce) {
+                    ANNOTATE_REDUCE_AFTER(reduce);
                     for (child_tid=tid+1; child_tid<=tid+thr_bar->leaf_kids; ++child_tid) {
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
                         (*reduce)(this_thr->th.th_local.reduce_data, other_threads[child_tid]->th.th_local.reduce_data);
                     }
+                    ANNOTATE_REDUCE_BEFORE(reduce);
+                    ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
                 }
                 (void) KMP_TEST_THEN_AND64((volatile kmp_int64 *)&thr_bar->b_arrived, ~(thr_bar->leaf_state)); // clear leaf_state bits
             }
@@ -799,7 +813,10 @@ __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr,
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
+                        ANNOTATE_REDUCE_AFTER(reduce);
                         (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
+                        ANNOTATE_REDUCE_BEFORE(reduce);
+                        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
                     }
                 }
             }
@@ -822,7 +839,10 @@ __kmp_hierarchical_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr,
                         KA_TRACE(100, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) += T#%d(%d:%d)\n",
                                        gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                                        team->t.t_id, child_tid));
+                        ANNOTATE_REDUCE_AFTER(reduce);
                         (*reduce)(this_thr->th.th_local.reduce_data, child_thr->th.th_local.reduce_data);
+                        ANNOTATE_REDUCE_BEFORE(reduce);
+                        ANNOTATE_REDUCE_BEFORE(&team->t.t_bar);
                     }
                 }
             }
@@ -863,7 +883,7 @@ __kmp_hierarchical_barrier_release(enum barrier_type bt, kmp_info_t *this_thr, i
                                    int propagate_icvs
                                    USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_hier_release);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_hier_release);
     register kmp_team_t *team;
     register kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
     register kmp_uint32 nproc;
@@ -1047,7 +1067,8 @@ int
 __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
               void *reduce_data, void (*reduce)(void *, void *))
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_barrier);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_plain_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
     register int tid = __kmp_tid_from_gtid(gtid);
     register kmp_info_t *this_thr = __kmp_threads[gtid];
     register kmp_team_t *team = this_thr->th.th_team;
@@ -1061,6 +1082,7 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
     KA_TRACE(15, ("__kmp_barrier: T#%d(%d:%d) has arrived\n",
                   gtid, __kmp_team_from_gtid(gtid)->t.t_id, __kmp_tid_from_gtid(gtid)));
 
+    ANNOTATE_NEW_BARRIER_BEGIN(&team->t.t_bar);
 #if OMPT_SUPPORT
     if (ompt_enabled) {
 #if OMPT_BLAME
@@ -1106,7 +1128,9 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
            the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical.
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
         }
 
@@ -1198,8 +1222,11 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
                     if( __itt_metadata_add_ptr ) {
                         // Initialize with master's wait time
                         kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
+                        // Set arrive time to zero to be able to check it in __kmp_invoke_task(); the same is done inside the loop below
+                        this_thr->th.th_bar_arrive_time = 0;
                         for (i=1; i<nproc; ++i) {
                             delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
+                            other_threads[i]->th.th_bar_arrive_time = 0;
                         }
                         __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, (kmp_uint64)( reduce != NULL));
                     }
@@ -1255,7 +1282,7 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
     } else { // Team is serialized.
         status = 0;
         if (__kmp_tasking_mode != tskm_immediate_exec) {
-#if OMP_41_ENABLED
+#if OMP_45_ENABLED
             if ( this_thr->th.th_task_team != NULL ) {
                 void *itt_sync_obj = NULL;
 #if USE_ITT_NOTIFY
@@ -1296,6 +1323,7 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
         this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
     }
 #endif
+    ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 
     return status;
 }
@@ -1304,11 +1332,13 @@ __kmp_barrier(enum barrier_type bt, int gtid, int is_split, size_t reduce_size,
 void
 __kmp_end_split_barrier(enum barrier_type bt, int gtid)
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_end_split_barrier);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_end_split_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(PLAIN_BARRIER);
     int tid = __kmp_tid_from_gtid(gtid);
     kmp_info_t *this_thr = __kmp_threads[gtid];
     kmp_team_t *team = this_thr->th.th_team;
 
+    ANNOTATE_NEW_BARRIER_BEGIN(&team->t.t_bar);
     if (!team->t.t_serialized) {
         if (KMP_MASTER_GTID(gtid)) {
             switch (__kmp_barrier_release_pattern[bt]) {
@@ -1339,13 +1369,15 @@ __kmp_end_split_barrier(enum barrier_type bt, int gtid)
             } // if
         }
     }
+    ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 }
 
 
 void
 __kmp_join_barrier(int gtid)
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_join_barrier);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_join_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
     register kmp_info_t *this_thr = __kmp_threads[gtid];
     register kmp_team_t *team;
     register kmp_uint nproc;
@@ -1388,7 +1420,8 @@ __kmp_join_barrier(int gtid)
     KMP_DEBUG_ASSERT(this_thr == team->t.t_threads[tid]);
     KA_TRACE(10, ("__kmp_join_barrier: T#%d(%d:%d) arrived at join barrier\n", gtid, team_id, tid));
 
-#if OMPT_SUPPORT 
+    ANNOTATE_NEW_BARRIER_BEGIN(&team->t.t_bar);
+#if OMPT_SUPPORT
 #if OMPT_TRACE
     if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_barrier_begin)) {
@@ -1418,7 +1451,9 @@ __kmp_join_barrier(int gtid)
        down EPCC parallel by 2x. As a workaround, we do not perform the copy if blocktime=infinite,
        since the values are not used by __kmp_wait_template() in that case. */
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
         this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
         this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
     }
 
@@ -1457,11 +1492,21 @@ __kmp_join_barrier(int gtid)
        the kmp_task_team_t structs.  */
     if (KMP_MASTER_TID(tid)) {
         if (__kmp_tasking_mode != tskm_immediate_exec) {
-            // Master shouldn't call decrease_load().         // TODO: enable master threads.
-            // Master should have th_may_decrease_load == 0.  // TODO: enable master threads.
             __kmp_task_team_wait(this_thr, team
                                  USE_ITT_BUILD_ARG(itt_sync_obj) );
         }
+#if KMP_STATS_ENABLED
+        // Have master thread flag the workers to indicate they are now waiting for
+        // next parallel region, Also wake them up so they switch their timers to idle.
+        for (int i=0; i<team->t.t_nproc; ++i) {
+            kmp_info_t* team_thread = team->t.t_threads[i];
+            if (team_thread == this_thr)
+                continue;
+            team_thread->th.th_stats->setIdleFlag();
+            if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && team_thread->th.th_sleep_loc != NULL)
+                __kmp_null_resume_wrapper(__kmp_gtid_from_thread(team_thread), team_thread->th.th_sleep_loc);
+        }
+#endif
 #if USE_ITT_BUILD
         if (__itt_sync_create_ptr || KMP_ITT_DEBUG)
             __kmp_itt_barrier_middle(gtid, itt_sync_obj);
@@ -1491,8 +1536,11 @@ __kmp_join_barrier(int gtid)
                 if( __itt_metadata_add_ptr ) {
                     // Initialize with master's wait time
                     kmp_uint64 delta = cur_time - this_thr->th.th_bar_arrive_time;
+                    // Set arrive time to zero to be able to check it in __kmp_invoke_task(); the same is done inside the loop below
+                    this_thr->th.th_bar_arrive_time = 0;
                     for (i=1; i<nproc; ++i) {
                         delta += ( cur_time - other_threads[i]->th.th_bar_arrive_time );
+                        other_threads[i]->th.th_bar_arrive_time = 0;
                     }
                     __kmp_itt_metadata_imbalance(gtid, this_thr->th.th_frame_time, cur_time, delta, 0);
                 }
@@ -1535,6 +1583,7 @@ __kmp_join_barrier(int gtid)
         this_thr->th.ompt_thread_info.state = ompt_state_overhead;
     }
 #endif
+    ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 }
 
 
@@ -1542,12 +1591,15 @@ __kmp_join_barrier(int gtid)
 void
 __kmp_fork_barrier(int gtid, int tid)
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_fork_barrier);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_fork_barrier);
+    KMP_SET_THREAD_STATE_BLOCK(FORK_JOIN_BARRIER);
     kmp_info_t *this_thr = __kmp_threads[gtid];
     kmp_team_t *team = (tid == 0) ? this_thr->th.th_team : NULL;
 #if USE_ITT_BUILD
     void * itt_sync_obj = NULL;
 #endif /* USE_ITT_BUILD */
+    if (team)
+      ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
 
     KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) has arrived\n",
                   gtid, (team != NULL) ? team->t.t_id : -1, tid));
@@ -1590,7 +1642,9 @@ __kmp_fork_barrier(int gtid, int tid)
            access it when the team struct is not guaranteed to exist. */
         // See note about the corresponding code in __kmp_join_barrier() being performance-critical
         if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
             this_thr->th.th_team_bt_intervals = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_intervals;
+#endif
             this_thr->th.th_team_bt_set = team->t.t_implicit_task_taskdata[tid].td_icvs.bt_set;
         }
     } // master
@@ -1651,7 +1705,7 @@ __kmp_fork_barrier(int gtid, int tid)
        the fixed ICVs in the master's thread struct, because it is not always the case that the
        threads arrays have been allocated when __kmp_fork_call() is executed. */
     {
-        KMP_TIME_DEVELOPER_BLOCK(USER_icv_copy);
+        KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_icv_copy);
         if (!KMP_MASTER_TID(tid)) {  // master thread already has ICVs
             // Copy the initial ICVs from the master's thread struct to the implicit task for this tid.
             KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d) is PULLing ICVs\n", gtid, tid));
@@ -1698,6 +1752,7 @@ __kmp_fork_barrier(int gtid, int tid)
         } // (prepare called inside barrier_release)
     }
 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
+    ANNOTATE_NEW_BARRIER_END(&team->t.t_bar);
     KA_TRACE(10, ("__kmp_fork_barrier: T#%d(%d:%d) is leaving\n", gtid, team->t.t_id, tid));
 }
 
@@ -1705,7 +1760,7 @@ __kmp_fork_barrier(int gtid, int tid)
 void
 __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc, kmp_internal_control_t *new_icvs, ident_t *loc )
 {
-    KMP_TIME_DEVELOPER_BLOCK(KMP_setup_icv_copy);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_setup_icv_copy);
 
     KMP_DEBUG_ASSERT(team && new_nproc && new_icvs);
     KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);

runtime/src/kmp_cancel.cpp

@@ -29,12 +29,12 @@ Request cancellation of the binding OpenMP region.
 */
 kmp_int32 __kmpc_cancel(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) {
     kmp_info_t *this_thr = __kmp_threads [ gtid ];
-    
+
     KC_TRACE( 10, ("__kmpc_cancel: T#%d request %d OMP_CANCELLATION=%d\n", gtid, cncl_kind, __kmp_omp_cancellation) );
 
     KMP_DEBUG_ASSERT(cncl_kind != cancel_noreq);
-    KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop || 
-                     cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup); 
+    KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop ||
+                     cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup);
     KMP_DEBUG_ASSERT(__kmp_get_gtid() == gtid);
 
     if (__kmp_omp_cancellation) {
@@ -49,7 +49,7 @@ kmp_int32 __kmpc_cancel(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) {
                 KMP_DEBUG_ASSERT(this_team);
                 kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(&(this_team->t.t_cancel_request), cancel_noreq, cncl_kind);
                 if (old == cancel_noreq || old == cncl_kind) {
-                    //printf("__kmpc_cancel: this_team->t.t_cancel_request=%d @ %p\n", 
+                    //printf("__kmpc_cancel: this_team->t.t_cancel_request=%d @ %p\n",
                     //       this_team->t.t_cancel_request, &(this_team->t.t_cancel_request));
                     // we do not have a cancellation request in this team or we do have one
                     // that matches the current request -> cancel
@@ -61,12 +61,12 @@ kmp_int32 __kmpc_cancel(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) {
             // cancellation requests for a task group
             // are handled through the taskgroup structure
             {
-                kmp_taskdata_t*  task; 
+                kmp_taskdata_t*  task;
                 kmp_taskgroup_t* taskgroup;
-                
+
                 task = this_thr->th.th_current_task;
                 KMP_DEBUG_ASSERT( task );
-                
+
                 taskgroup = task->td_taskgroup;
                 if (taskgroup) {
                     kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(&(taskgroup->cancel_request), cancel_noreq, cncl_kind);
@@ -100,7 +100,7 @@ kmp_int32 __kmpc_cancel(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 cncl_kind) {
 @param gtid Global thread ID of encountering thread
 @param cncl_kind Cancellation kind (parallel, for, sections, taskgroup)
 
-@return returns true if a matching cancellation request has been flagged in the RTL and the 
+@return returns true if a matching cancellation request has been flagged in the RTL and the
 encountering thread has to cancel..
 
 Cancellation point for the encountering thread.
@@ -111,8 +111,8 @@ kmp_int32 __kmpc_cancellationpoint(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 c
     KC_TRACE( 10, ("__kmpc_cancellationpoint: T#%d request %d OMP_CANCELLATION=%d\n", gtid, cncl_kind, __kmp_omp_cancellation) );
 
     KMP_DEBUG_ASSERT(cncl_kind != cancel_noreq);
-    KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop || 
-                     cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup); 
+    KMP_DEBUG_ASSERT(cncl_kind == cancel_parallel || cncl_kind == cancel_loop ||
+                     cncl_kind == cancel_sections || cncl_kind == cancel_taskgroup);
     KMP_DEBUG_ASSERT(__kmp_get_gtid() == gtid);
 
     if (__kmp_omp_cancellation) {
@@ -144,15 +144,15 @@ kmp_int32 __kmpc_cancellationpoint(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 c
             // cancellation requests for a task group
             // are handled through the taskgroup structure
             {
-                kmp_taskdata_t*  task; 
+                kmp_taskdata_t*  task;
                 kmp_taskgroup_t* taskgroup;
-                
+
                 task = this_thr->th.th_current_task;
                 KMP_DEBUG_ASSERT( task );
-                
+
                 taskgroup = task->td_taskgroup;
                 if (taskgroup) {
-                    // return the current status of cancellation for the 
+                    // return the current status of cancellation for the
                     // taskgroup
                     return !!taskgroup->cancel_request;
                 }
@@ -178,11 +178,11 @@ kmp_int32 __kmpc_cancellationpoint(ident_t* loc_ref, kmp_int32 gtid, kmp_int32 c
 @param loc_ref location of the original task directive
 @param gtid Global thread ID of encountering thread
 
-@return returns true if a matching cancellation request has been flagged in the RTL and the 
+@return returns true if a matching cancellation request has been flagged in the RTL and the
 encountering thread has to cancel..
 
 Barrier with cancellation point to send threads from the barrier to the
-end of the parallel region.  Needs a special code pattern as documented 
+end of the parallel region.  Needs a special code pattern as documented
 in the design document for the cancellation feature.
 */
 kmp_int32
@@ -208,7 +208,7 @@ __kmpc_cancel_barrier(ident_t *loc, kmp_int32 gtid) {
             __kmpc_barrier(loc, gtid);
             this_team->t.t_cancel_request = cancel_noreq;
             // the next barrier is the fork/join barrier, which
-            // synchronizes the threads leaving here        
+            // synchronizes the threads leaving here
             break;
         case cancel_loop:
         case cancel_sections:
@@ -233,7 +233,7 @@ __kmpc_cancel_barrier(ident_t *loc, kmp_int32 gtid) {
             KMP_ASSERT ( 0 /* false */);
         }
     }
-    
+
     return ret;
 }
 
@@ -242,21 +242,21 @@ __kmpc_cancel_barrier(ident_t *loc, kmp_int32 gtid) {
 @param loc_ref location of the original task directive
 @param gtid Global thread ID of encountering thread
 
-@return returns true if a matching cancellation request has been flagged in the RTL and the 
+@return returns true if a matching cancellation request has been flagged in the RTL and the
 encountering thread has to cancel..
 
 Query function to query the current status of cancellation requests.
 Can be used to implement the following pattern:
- 
+
 if (kmp_get_cancellation_status(kmp_cancel_parallel)) {
     perform_cleanup();
-    #pragma omp cancellation point parallel      
+    #pragma omp cancellation point parallel
 }
 */
 int __kmp_get_cancellation_status(int cancel_kind) {
     if (__kmp_omp_cancellation) {
         kmp_info_t *this_thr = __kmp_entry_thread();
-        
+
         switch (cancel_kind) {
         case cancel_parallel:
         case cancel_loop:
@@ -267,7 +267,7 @@ int __kmp_get_cancellation_status(int cancel_kind) {
             }
         case cancel_taskgroup:
             {
-                kmp_taskdata_t*  task; 
+                kmp_taskdata_t*  task;
                 kmp_taskgroup_t* taskgroup;
                 task = this_thr->th.th_current_task;
                 taskgroup = task->td_taskgroup;

runtime/src/kmp_config.h.cmake

@@ -65,9 +65,13 @@
 #define KMP_VERSION_MINOR @LIBOMP_VERSION_MINOR@
 #define LIBOMP_OMP_VERSION @LIBOMP_OMP_VERSION@
 #define OMP_50_ENABLED (LIBOMP_OMP_VERSION >= 50)
-#define OMP_41_ENABLED (LIBOMP_OMP_VERSION >= 41)
+#define OMP_45_ENABLED (LIBOMP_OMP_VERSION >= 45)
 #define OMP_40_ENABLED (LIBOMP_OMP_VERSION >= 40)
 #define OMP_30_ENABLED (LIBOMP_OMP_VERSION >= 30)
+#cmakedefine01 LIBOMP_TSAN_SUPPORT
+#if LIBOMP_TSAN_SUPPORT
+#define TSAN_SUPPORT
+#endif
 
 // Configured cache line based on architecture
 #if KMP_ARCH_PPC64
@@ -85,7 +89,7 @@
 #define KMP_ADJUST_BLOCKTIME 1
 #define BUILD_PARALLEL_ORDERED 1
 #define KMP_ASM_INTRINS 1
-#define USE_ITT_BUILD 1
+#define USE_ITT_BUILD LIBOMP_USE_ITT_NOTIFY
 #define INTEL_ITTNOTIFY_PREFIX __kmp_itt_
 #if ! KMP_MIC
 # define USE_LOAD_BALANCE 1

runtime/src/kmp_csupport.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_csupport.c -- kfront linkage support for OpenMP.
+ * kmp_csupport.cpp -- kfront linkage support for OpenMP.
  */
 
 
@@ -17,6 +17,7 @@
 #include "kmp.h"
 #include "kmp_i18n.h"
 #include "kmp_itt.h"
+#include "kmp_lock.h"
 #include "kmp_error.h"
 #include "kmp_stats.h"
 
@@ -120,9 +121,9 @@ waiting for work.
 kmp_int32
 __kmpc_global_num_threads(ident_t *loc)
 {
-    KC_TRACE( 10, ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
+    KC_TRACE(10,("__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
 
-    return TCR_4(__kmp_nth);
+    return TCR_4(__kmp_all_nth);
 }
 
 /*!
@@ -282,7 +283,7 @@ __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
 {
   int         gtid = __kmp_entry_gtid();
 
-#if (KMP_STATS_ENABLED)  
+#if (KMP_STATS_ENABLED)
   int inParallel = __kmpc_in_parallel(loc);
   if (inParallel)
   {
@@ -290,7 +291,6 @@ __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
   }
   else
   {
-      KMP_STOP_EXPLICIT_TIMER(OMP_serial);
       KMP_COUNT_BLOCK(OMP_PARALLEL);
   }
 #endif
@@ -301,12 +301,20 @@ __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
     va_start(   ap, microtask );
 
 #if OMPT_SUPPORT
-    int tid = __kmp_tid_from_gtid( gtid );
-    kmp_info_t *master_th = __kmp_threads[ gtid ];
-    kmp_team_t *parent_team = master_th->th.th_team;
+    ompt_frame_t* ompt_frame;
     if (ompt_enabled) {
-       parent_team->t.t_implicit_task_taskdata[tid].
-           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
+       kmp_info_t *master_th = __kmp_threads[ gtid ];
+       kmp_team_t *parent_team = master_th->th.th_team;
+       ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
+       if (lwt)
+         ompt_frame = &(lwt->ompt_task_info.frame);
+       else
+       {
+         int tid = __kmp_tid_from_gtid( gtid );
+         ompt_frame = &(parent_team->t.t_implicit_task_taskdata[tid].
+         ompt_task_info.frame);
+       }
+       ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
@@ -338,17 +346,7 @@ __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
 
     va_end( ap );
 
-#if OMPT_SUPPORT
-    if (ompt_enabled) {
-        parent_team->t.t_implicit_task_taskdata[tid].
-            ompt_task_info.frame.reenter_runtime_frame = 0;
-    }
-#endif
   }
-#if (KMP_STATS_ENABLED)  
-  if (!inParallel)
-      KMP_START_EXPLICIT_TIMER(OMP_serial);
-#endif
 }
 
 #if OMP_40_ENABLED
@@ -400,7 +398,7 @@ __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
     int tid = __kmp_tid_from_gtid( gtid );
     if (ompt_enabled) {
         parent_team->t.t_implicit_task_taskdata[tid].
-           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
+           ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
@@ -431,13 +429,6 @@ __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
 #endif
     );
 
-#if OMPT_SUPPORT
-    if (ompt_enabled) {
-        parent_team->t.t_implicit_task_taskdata[tid].
-           ompt_task_info.frame.reenter_runtime_frame = NULL;
-    }
-#endif
-
     this_thr->th.th_teams_microtask = NULL;
     this_thr->th.th_teams_level = 0;
     *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
@@ -473,9 +464,10 @@ when the condition is false.
 void
 __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
 {
-    __kmp_serialized_parallel(loc, global_tid); /* The implementation is now in kmp_runtime.c so that it can share static functions with
-                                                 * kmp_fork_call since the tasks to be done are similar in each case.
-                                                 */
+    // The implementation is now in kmp_runtime.cpp so that it can share static
+    // functions with kmp_fork_call since the tasks to be done are similar in
+    // each case.
+    __kmp_serialized_parallel(loc, global_tid);
 }
 
 /*!
@@ -506,12 +498,12 @@ __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
     this_thr    = __kmp_threads[ global_tid ];
     serial_team = this_thr->th.th_serial_team;
 
-   #if OMP_41_ENABLED
+   #if OMP_45_ENABLED
    kmp_task_team_t *   task_team = this_thr->th.th_task_team;
 
    // we need to wait for the proxy tasks before finishing the thread
    if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
-        __kmp_task_team_wait(this_thr, serial_team, NULL ); // is an ITT object needed here?
+        __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL) ); // is an ITT object needed here?
    #endif
 
     KMP_MB();
@@ -585,36 +577,6 @@ __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
         }
     }
 
-#if USE_ITT_BUILD
-    kmp_uint64 cur_time = 0;
-#if  USE_ITT_NOTIFY
-    if ( __itt_get_timestamp_ptr ) {
-        cur_time = __itt_get_timestamp();
-    }
-#endif /* USE_ITT_NOTIFY */
-    if ( this_thr->th.th_team->t.t_level == 0
-#if OMP_40_ENABLED
-        && this_thr->th.th_teams_microtask == NULL
-#endif
-    ) {
-        // Report the barrier
-        this_thr->th.th_ident = loc;
-        if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
-            ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
-        {
-            __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized,
-                                    cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
-            if ( __kmp_forkjoin_frames_mode == 3 )
-                // Since barrier frame for serialized region is equal to the region we use the same begin timestamp as for the barrier.
-                __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time,
-                                        cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
-        } else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
-            ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
-            // Mark the end of the "parallel" region for VTune. Only use one of frame notification scheme at the moment.
-            __kmp_itt_region_joined( global_tid, 1 );
-    }
-#endif /* USE_ITT_BUILD */
-
     if ( __kmp_env_consistency_check )
         __kmp_pop_parallel( global_tid, NULL );
 }
@@ -654,14 +616,16 @@ __kmpc_flush(ident_t *loc)
             if ( ! __kmp_cpuinfo.sse2 ) {
                 // CPU cannot execute SSE2 instructions.
             } else {
-                #if KMP_COMPILER_ICC || KMP_COMPILER_MSVC
+                #if KMP_COMPILER_ICC
                 _mm_mfence();
+                #elif KMP_COMPILER_MSVC
+                MemoryBarrier();
                 #else
                 __sync_synchronize();
                 #endif // KMP_COMPILER_ICC
             }; // if
         #endif // KMP_MIC
-    #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+    #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64)
         // Nothing to see here move along
     #elif KMP_ARCH_PPC64
         // Nothing needed here (we have a real MB above).
@@ -697,7 +661,6 @@ void
 __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
 {
     KMP_COUNT_BLOCK(OMP_BARRIER);
-    KMP_TIME_BLOCK(OMP_barrier);
     KC_TRACE( 10, ("__kmpc_barrier: called T#%d\n", global_tid ) );
 
     if (! TCR_4(__kmp_init_parallel))
@@ -711,6 +674,14 @@ __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
         __kmp_check_barrier( global_tid, ct_barrier, loc );
     }
 
+#if OMPT_SUPPORT && OMPT_TRACE
+    ompt_frame_t * ompt_frame;
+    if (ompt_enabled ) {
+        ompt_frame = __ompt_get_task_frame_internal(0);
+        if ( ompt_frame->reenter_runtime_frame == NULL )
+            ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
+    }
+#endif
     __kmp_threads[ global_tid ]->th.th_ident = loc;
     // TODO: explicit barrier_wait_id:
     //   this function is called when 'barrier' directive is present or
@@ -720,6 +691,11 @@ __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
     // 4) no sync is required
 
     __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
+#if OMPT_SUPPORT && OMPT_TRACE
+    if (ompt_enabled ) {
+        ompt_frame->reenter_runtime_frame = NULL;
+    }
+#endif
 }
 
 /* The BARRIER for a MASTER section is always explicit   */
@@ -732,7 +708,6 @@ __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
 kmp_int32
 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
 {
-    KMP_COUNT_BLOCK(OMP_MASTER);
     int status = 0;
 
     KC_TRACE( 10, ("__kmpc_master: called T#%d\n", global_tid ) );
@@ -741,7 +716,8 @@ __kmpc_master(ident_t *loc, kmp_int32 global_tid)
         __kmp_parallel_initialize();
 
     if( KMP_MASTER_GTID( global_tid )) {
-        KMP_START_EXPLICIT_TIMER(OMP_master);
+        KMP_COUNT_BLOCK(OMP_MASTER);
+        KMP_PUSH_PARTITIONED_TIMER(OMP_master);
         status = 1;
     }
 
@@ -791,7 +767,7 @@ __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
     KC_TRACE( 10, ("__kmpc_end_master: called T#%d\n", global_tid ) );
 
     KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
-    KMP_STOP_EXPLICIT_TIMER(OMP_master);
+    KMP_POP_PARTITIONED_TIMER();
 
 #if OMPT_SUPPORT && OMPT_TRACE
     kmp_info_t  *this_thr        = __kmp_threads[ global_tid ];
@@ -955,8 +931,10 @@ __kmp_init_indirect_csptr(kmp_critical_name * crit, ident_t const * loc, kmp_int
         } else {                                                                                                 \
             KMP_YIELD_SPIN(spins);                                                                               \
         }                                                                                                        \
+        kmp_backoff_t backoff = __kmp_spin_backoff_params;                                                       \
         while (l->lk.poll != KMP_LOCK_FREE(tas) ||                                                               \
                ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) {   \
+            __kmp_spin_backoff(&backoff);                                                                        \
             if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {                        \
                 KMP_YIELD(TRUE);                                                                                 \
             } else {                                                                                             \
@@ -1019,7 +997,7 @@ __kmp_init_indirect_csptr(kmp_critical_name * crit, ident_t const * loc, kmp_int
 // Fast-path test futex lock
 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) {                                                                       \
     kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock;                                                               \
-    if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1, futex) << 1)) {    \
+    if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1 << 1, futex))) {    \
         KMP_FSYNC_ACQUIRED(ftx);                                                                                    \
         rc = TRUE;                                                                                                  \
     } else {                                                                                                        \
@@ -1114,6 +1092,7 @@ __kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit )
     __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
 #else
     KMP_COUNT_BLOCK(OMP_CRITICAL);
+    KMP_TIME_PARTITIONED_BLOCK(OMP_critical_wait);        /* Time spent waiting to enter the critical section */
     kmp_user_lock_p lck;
 
     KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) );
@@ -1126,7 +1105,7 @@ __kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit )
       && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
         lck = (kmp_user_lock_p)crit;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
         lck = (kmp_user_lock_p)crit;
@@ -1155,6 +1134,7 @@ __kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit )
     __kmp_itt_critical_acquired( lck );
 #endif /* USE_ITT_BUILD */
 
+    KMP_START_EXPLICIT_TIMER(OMP_critical);
     KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
 #endif // KMP_USE_DYNAMIC_LOCK
 }
@@ -1170,13 +1150,20 @@ __kmp_map_hint_to_lock(uintptr_t hint)
 #else
 # define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
 #endif
+
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+# define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm)
+#else
+# define KMP_CPUINFO_RTM 0
+#endif
+
     // Hints that do not require further logic
     if (hint & kmp_lock_hint_hle)
         return KMP_TSX_LOCK(hle);
     if (hint & kmp_lock_hint_rtm)
-        return (__kmp_cpuinfo.rtm)? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq;
+        return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq;
     if (hint & kmp_lock_hint_adaptive)
-        return (__kmp_cpuinfo.rtm)? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq;
+        return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq;
 
     // Rule out conflicting hints first by returning the default lock
     if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
@@ -1267,6 +1254,7 @@ __kmpc_critical_with_hint( ident_t * loc, kmp_int32 global_tid, kmp_critical_nam
     __kmp_itt_critical_acquired( lck );
 #endif /* USE_ITT_BUILD */
 
+    KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
     KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
 } // __kmpc_critical_with_hint
 
@@ -1329,7 +1317,7 @@ __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
       && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
         lck = (kmp_user_lock_p)crit;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
         lck = (kmp_user_lock_p)crit;
@@ -1359,7 +1347,7 @@ __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
 #endif
 
 #endif // KMP_USE_DYNAMIC_LOCK
-
+    KMP_POP_PARTITIONED_TIMER();
     KA_TRACE( 15, ("__kmpc_end_critical: done T#%d\n", global_tid ));
 }
 
@@ -1476,10 +1464,12 @@ introduce an explicit barrier if it is required.
 kmp_int32
 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
 {
-    KMP_COUNT_BLOCK(OMP_SINGLE);
     kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
-    if(rc == TRUE) {
-        KMP_START_EXPLICIT_TIMER(OMP_single);
+
+    if (rc) {
+        // We are going to execute the single statement, so we should count it.
+        KMP_COUNT_BLOCK(OMP_SINGLE);
+        KMP_PUSH_PARTITIONED_TIMER(OMP_single);
     }
 
 #if OMPT_SUPPORT && OMPT_TRACE
@@ -1522,7 +1512,7 @@ void
 __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
 {
     __kmp_exit_single( global_tid );
-    KMP_STOP_EXPLICIT_TIMER(OMP_single);
+    KMP_POP_PARTITIONED_TIMER();
 
 #if OMPT_SUPPORT && OMPT_TRACE
     kmp_info_t *this_thr        = __kmp_threads[ global_tid ];
@@ -1553,13 +1543,10 @@ __kmpc_for_static_fini( ident_t *loc, kmp_int32 global_tid )
 #if OMPT_SUPPORT && OMPT_TRACE
     if (ompt_enabled &&
         ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
-        kmp_info_t *this_thr = __kmp_threads[ global_tid ];
-        kmp_team_t *team     = this_thr -> th.th_team;
-        int tid = __kmp_tid_from_gtid( global_tid );
-
+        ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
+        ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
         ompt_callbacks.ompt_callback(ompt_event_loop_end)(
-            team->t.ompt_team_info.parallel_id,
-            team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
+            team_info->parallel_id, task_info->task_id);
     }
 #endif
 
@@ -1675,6 +1662,15 @@ kmpc_set_defaults( char const * str )
     __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
 }
 
+void
+kmpc_set_disp_num_buffers( int arg )
+{
+    // ignore after initialization because some teams have already
+    // allocated dispatch buffers
+    if( __kmp_init_serial == 0 && arg > 0 )
+        __kmp_dispatch_num_buffers = arg;
+}
+
 int
 kmpc_set_affinity_mask_proc( int proc, void **mask )
 {
@@ -1926,7 +1922,7 @@ __kmpc_init_lock( ident_t * loc, kmp_int32 gtid,  void ** user_lock ) {
       && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
@@ -1981,7 +1977,7 @@ __kmpc_init_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -2030,7 +2026,7 @@ __kmpc_destroy_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
@@ -2056,7 +2052,7 @@ __kmpc_destroy_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         ;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         ;
@@ -2087,7 +2083,7 @@ __kmpc_destroy_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -2115,7 +2111,7 @@ __kmpc_destroy_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
      + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
         ;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -2160,7 +2156,7 @@ __kmpc_set_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
@@ -2216,7 +2212,7 @@ __kmpc_set_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
       + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -2295,7 +2291,7 @@ __kmpc_unset_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
         lck = (kmp_user_lock_p)user_lock;
 #endif
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
@@ -2355,7 +2351,7 @@ __kmpc_unset_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
         lck = (kmp_user_lock_p)user_lock;
 #endif
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -2434,7 +2430,7 @@ __kmpc_test_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
       && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
       && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
@@ -2491,7 +2487,7 @@ __kmpc_test_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
       + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
         lck = (kmp_user_lock_p)user_lock;
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     else if ( ( __kmp_user_lock_kind == lk_futex )
      && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
      <= OMP_NEST_LOCK_T_SIZE ) ) {
@@ -3049,5 +3045,294 @@ void __kmpc_place_threads(int nS, int sO, int nC, int cO, int nT)
     __kmp_place_num_threads_per_core = nT;
 }
 
+#if OMP_45_ENABLED
+/*!
+@ingroup WORK_SHARING
+@param loc  source location information.
+@param gtid  global thread number.
+@param num_dims  number of associated doacross loops.
+@param dims  info on loops bounds.
+
+Initialize doacross loop information.
+Expect compiler send us inclusive bounds,
+e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2.
+*/
+void
+__kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, struct kmp_dim * dims)
+{
+    int j, idx;
+    kmp_int64 last, trace_count;
+    kmp_info_t *th = __kmp_threads[gtid];
+    kmp_team_t *team = th->th.th_team;
+    kmp_uint32 *flags;
+    kmp_disp_t *pr_buf = th->th.th_dispatch;
+    dispatch_shared_info_t *sh_buf;
+
+    KA_TRACE(20,("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
+                 gtid, num_dims, !team->t.t_serialized));
+    KMP_DEBUG_ASSERT(dims != NULL);
+    KMP_DEBUG_ASSERT(num_dims > 0);
+
+    if( team->t.t_serialized ) {
+        KA_TRACE(20,("__kmpc_doacross_init() exit: serialized team\n"));
+        return; // no dependencies if team is serialized
+    }
+    KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
+    idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for the next loop
+    sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
+
+    // Save bounds info into allocated private buffer
+    KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
+    pr_buf->th_doacross_info =
+        (kmp_int64*)__kmp_thread_malloc(th, sizeof(kmp_int64)*(4 * num_dims + 1));
+    KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
+    pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions
+    // Save also address of num_done in order to access it later without knowing the buffer index
+    pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
+    pr_buf->th_doacross_info[2] = dims[0].lo;
+    pr_buf->th_doacross_info[3] = dims[0].up;
+    pr_buf->th_doacross_info[4] = dims[0].st;
+    last = 5;
+    for( j = 1; j < num_dims; ++j ) {
+        kmp_int64 range_length; // To keep ranges of all dimensions but the first dims[0]
+        if( dims[j].st == 1 ) { // most common case
+            // AC: should we care of ranges bigger than LLONG_MAX? (not for now)
+            range_length = dims[j].up - dims[j].lo + 1;
+        } else {
+            if( dims[j].st > 0 ) {
+                KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
+                range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
+            } else {            // negative increment
+                KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
+                range_length = (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
+            }
+        }
+        pr_buf->th_doacross_info[last++] = range_length;
+        pr_buf->th_doacross_info[last++] = dims[j].lo;
+        pr_buf->th_doacross_info[last++] = dims[j].up;
+        pr_buf->th_doacross_info[last++] = dims[j].st;
+    }
+
+    // Compute total trip count.
+    // Start with range of dims[0] which we don't need to keep in the buffer.
+    if( dims[0].st == 1 ) { // most common case
+        trace_count = dims[0].up - dims[0].lo + 1;
+    } else if( dims[0].st > 0 ) {
+        KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
+        trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
+    } else {   // negative increment
+        KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
+        trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
+    }
+    for( j = 1; j < num_dims; ++j ) {
+        trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
+    }
+    KMP_DEBUG_ASSERT(trace_count > 0);
+
+    // Check if shared buffer is not occupied by other loop (idx - __kmp_dispatch_num_buffers)
+    if( idx != sh_buf->doacross_buf_idx ) {
+        // Shared buffer is occupied, wait for it to be free
+        __kmp_wait_yield_4( (kmp_uint32*)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL );
+    }
+    // Check if we are the first thread. After the CAS the first thread gets 0,
+    // others get 1 if initialization is in progress, allocated pointer otherwise.
+    flags = (kmp_uint32*)KMP_COMPARE_AND_STORE_RET64(
+        (kmp_int64*)&sh_buf->doacross_flags,NULL,(kmp_int64)1);
+    if( flags == NULL ) {
+        // we are the first thread, allocate the array of flags
+        kmp_int64 size = trace_count / 8 + 8; // in bytes, use single bit per iteration
+        sh_buf->doacross_flags = (kmp_uint32*)__kmp_thread_calloc(th, size, 1);
+    } else if( (kmp_int64)flags == 1 ) {
+        // initialization is still in progress, need to wait
+        while( (volatile kmp_int64)sh_buf->doacross_flags == 1 ) {
+            KMP_YIELD(TRUE);
+        }
+    }
+    KMP_DEBUG_ASSERT((kmp_int64)sh_buf->doacross_flags > 1); // check value of pointer
+    pr_buf->th_doacross_flags = sh_buf->doacross_flags;      // save private copy in order to not
+                                                             // touch shared buffer on each iteration
+    KA_TRACE(20,("__kmpc_doacross_init() exit: T#%d\n", gtid));
+}
+
+void
+__kmpc_doacross_wait(ident_t *loc, int gtid, long long *vec)
+{
+    kmp_int32 shft, num_dims, i;
+    kmp_uint32 flag;
+    kmp_int64 iter_number; // iteration number of "collapsed" loop nest
+    kmp_info_t *th = __kmp_threads[gtid];
+    kmp_team_t *team = th->th.th_team;
+    kmp_disp_t *pr_buf;
+    kmp_int64 lo, up, st;
+
+    KA_TRACE(20,("__kmpc_doacross_wait() enter: called T#%d\n", gtid));
+    if( team->t.t_serialized ) {
+        KA_TRACE(20,("__kmpc_doacross_wait() exit: serialized team\n"));
+        return; // no dependencies if team is serialized
+    }
+
+    // calculate sequential iteration number and check out-of-bounds condition
+    pr_buf = th->th.th_dispatch;
+    KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
+    num_dims = pr_buf->th_doacross_info[0];
+    lo = pr_buf->th_doacross_info[2];
+    up = pr_buf->th_doacross_info[3];
+    st = pr_buf->th_doacross_info[4];
+    if( st == 1 ) { // most common case
+        if( vec[0] < lo || vec[0] > up ) {
+            KA_TRACE(20,(
+                "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                gtid, vec[0], lo, up));
+            return;
+        }
+        iter_number = vec[0] - lo;
+    } else if( st > 0 ) {
+        if( vec[0] < lo || vec[0] > up ) {
+            KA_TRACE(20,(
+                "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                gtid, vec[0], lo, up));
+            return;
+        }
+        iter_number = (kmp_uint64)(vec[0] - lo) / st;
+    } else {        // negative increment
+        if( vec[0] > lo || vec[0] < up ) {
+            KA_TRACE(20,(
+                "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                gtid, vec[0], lo, up));
+            return;
+        }
+        iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
+    }
+    for( i = 1; i < num_dims; ++i ) {
+        kmp_int64 iter, ln;
+        kmp_int32 j = i * 4;
+        ln = pr_buf->th_doacross_info[j + 1];
+        lo = pr_buf->th_doacross_info[j + 2];
+        up = pr_buf->th_doacross_info[j + 3];
+        st = pr_buf->th_doacross_info[j + 4];
+        if( st == 1 ) {
+            if( vec[i] < lo || vec[i] > up ) {
+                KA_TRACE(20,(
+                    "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                    gtid, vec[i], lo, up));
+                return;
+            }
+            iter = vec[i] - lo;
+        } else if( st > 0 ) {
+            if( vec[i] < lo || vec[i] > up ) {
+                KA_TRACE(20,(
+                    "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                    gtid, vec[i], lo, up));
+                return;
+            }
+            iter = (kmp_uint64)(vec[i] - lo) / st;
+        } else {   // st < 0
+            if( vec[i] > lo || vec[i] < up ) {
+                KA_TRACE(20,(
+                    "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
+                    gtid, vec[i], lo, up));
+                return;
+            }
+            iter = (kmp_uint64)(lo - vec[i]) / (-st);
+        }
+        iter_number = iter + ln * iter_number;
+    }
+    shft = iter_number % 32; // use 32-bit granularity
+    iter_number >>= 5;       // divided by 32
+    flag = 1 << shft;
+    while( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 ) {
+        KMP_YIELD(TRUE);
+    }
+    KA_TRACE(20,("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
+                 gtid, (iter_number<<5)+shft));
+}
+
+void
+__kmpc_doacross_post(ident_t *loc, int gtid, long long *vec)
+{
+    kmp_int32 shft, num_dims, i;
+    kmp_uint32 flag;
+    kmp_int64 iter_number; // iteration number of "collapsed" loop nest
+    kmp_info_t *th = __kmp_threads[gtid];
+    kmp_team_t *team = th->th.th_team;
+    kmp_disp_t *pr_buf;
+    kmp_int64 lo, st;
+
+    KA_TRACE(20,("__kmpc_doacross_post() enter: called T#%d\n", gtid));
+    if( team->t.t_serialized ) {
+        KA_TRACE(20,("__kmpc_doacross_post() exit: serialized team\n"));
+        return; // no dependencies if team is serialized
+    }
+
+    // calculate sequential iteration number (same as in "wait" but no out-of-bounds checks)
+    pr_buf = th->th.th_dispatch;
+    KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
+    num_dims = pr_buf->th_doacross_info[0];
+    lo = pr_buf->th_doacross_info[2];
+    st = pr_buf->th_doacross_info[4];
+    if( st == 1 ) { // most common case
+        iter_number = vec[0] - lo;
+    } else if( st > 0 ) {
+        iter_number = (kmp_uint64)(vec[0] - lo) / st;
+    } else {        // negative increment
+        iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
+    }
+    for( i = 1; i < num_dims; ++i ) {
+        kmp_int64 iter, ln;
+        kmp_int32 j = i * 4;
+        ln = pr_buf->th_doacross_info[j + 1];
+        lo = pr_buf->th_doacross_info[j + 2];
+        st = pr_buf->th_doacross_info[j + 4];
+        if( st == 1 ) {
+            iter = vec[i] - lo;
+        } else if( st > 0 ) {
+            iter = (kmp_uint64)(vec[i] - lo) / st;
+        } else {   // st < 0
+            iter = (kmp_uint64)(lo - vec[i]) / (-st);
+        }
+        iter_number = iter + ln * iter_number;
+    }
+    shft = iter_number % 32; // use 32-bit granularity
+    iter_number >>= 5;       // divided by 32
+    flag = 1 << shft;
+    if( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 )
+        KMP_TEST_THEN_OR32( (kmp_int32*)&pr_buf->th_doacross_flags[iter_number], (kmp_int32)flag );
+    KA_TRACE(20,("__kmpc_doacross_post() exit: T#%d iter %lld posted\n",
+                 gtid, (iter_number<<5)+shft));
+}
+
+void
+__kmpc_doacross_fini(ident_t *loc, int gtid)
+{
+    kmp_int64 num_done;
+    kmp_info_t *th = __kmp_threads[gtid];
+    kmp_team_t *team = th->th.th_team;
+    kmp_disp_t *pr_buf = th->th.th_dispatch;
+
+    KA_TRACE(20,("__kmpc_doacross_fini() enter: called T#%d\n", gtid));
+    if( team->t.t_serialized ) {
+        KA_TRACE(20,("__kmpc_doacross_fini() exit: serialized team %p\n", team));
+        return; // nothing to do
+    }
+    num_done = KMP_TEST_THEN_INC64((kmp_int64*)pr_buf->th_doacross_info[1]) + 1;
+    if( num_done == th->th.th_team_nproc ) {
+        // we are the last thread, need to free shared resources
+        int idx = pr_buf->th_doacross_buf_idx - 1;
+        dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
+        KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done);
+        KMP_DEBUG_ASSERT(num_done == (kmp_int64)sh_buf->doacross_num_done);
+        KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
+        __kmp_thread_free(th, (void*)sh_buf->doacross_flags);
+        sh_buf->doacross_flags = NULL;
+        sh_buf->doacross_num_done = 0;
+        sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use
+    }
+    // free private resources (need to keep buffer index forever)
+    __kmp_thread_free(th, (void*)pr_buf->th_doacross_info);
+    pr_buf->th_doacross_info = NULL;
+    KA_TRACE(20,("__kmpc_doacross_fini() exit: T#%d\n", gtid));
+}
+#endif
+
 // end of file //
 

runtime/src/kmp_debug.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_debug.c -- debug utilities for the Guide library
+ * kmp_debug.cpp -- debug utilities for the Guide library
  */
 
 

runtime/src/kmp_debug.h

@@ -42,7 +42,7 @@
 
 #define __KMP_BUILD_ASSERT( expr, suffix )  typedef char __kmp_build_check_##suffix[ (expr) ? 1 : -1 ]
 #define _KMP_BUILD_ASSERT( expr, suffix )   __KMP_BUILD_ASSERT( (expr), suffix )
-#ifdef KMP_USE_ASSERT 
+#ifdef KMP_USE_ASSERT
     #define KMP_BUILD_ASSERT( expr )            _KMP_BUILD_ASSERT( (expr), __LINE__ )
 #else
     #define KMP_BUILD_ASSERT( expr )            /* nothing to do */

runtime/src/kmp_debugger.cpp

@@ -93,7 +93,7 @@ __kmp_omp_debug_struct_info = {
     sizeof( void * ),
     OMP_LOCK_T_SIZE < sizeof(void *),
     bs_last_barrier,
-    TASK_DEQUE_SIZE,
+    INITIAL_TASK_DEQUE_SIZE,
 
     // thread structure information
     sizeof( kmp_base_info_t ),
@@ -222,6 +222,7 @@ __kmp_omp_debug_struct_info = {
     // thread_data_t.
     sizeof( kmp_thread_data_t ),
     offset_and_size_of( kmp_base_thread_data_t, td_deque             ),
+    offset_and_size_of( kmp_base_thread_data_t, td_deque_size        ),
     offset_and_size_of( kmp_base_thread_data_t, td_deque_head        ),
     offset_and_size_of( kmp_base_thread_data_t, td_deque_tail        ),
     offset_and_size_of( kmp_base_thread_data_t, td_deque_ntasks      ),

runtime/src/kmp_debugger.h

@@ -23,19 +23,19 @@
 
 /* * This external variable can be set by any debugger to flag to the runtime that we
    are currently executing inside a debugger.  This will allow the debugger to override
-   the number of threads spawned in a parallel region by using __kmp_omp_num_threads() (below). 
-   * When __kmp_debugging is TRUE, each team and each task gets a unique integer identifier 
+   the number of threads spawned in a parallel region by using __kmp_omp_num_threads() (below).
+   * When __kmp_debugging is TRUE, each team and each task gets a unique integer identifier
    that can be used by debugger to conveniently identify teams and tasks.
    * The debugger has access to __kmp_omp_debug_struct_info which contains information
-   about the OpenMP library's important internal structures.  This access will allow the debugger 
+   about the OpenMP library's important internal structures.  This access will allow the debugger
    to read detailed information from the typical OpenMP constructs (teams, threads, tasking, etc. )
    during a debugging session and offer detailed and useful information which the user can probe
    about the OpenMP portion of their code.
    */
 extern int __kmp_debugging;             /* Boolean whether currently debugging OpenMP RTL */
 // Return number of threads specified by the debugger for given parallel region.
-/* The ident field, which represents a source file location, is used to check if the 
-   debugger has changed the number of threads for the parallel region at source file 
+/* The ident field, which represents a source file location, is used to check if the
+   debugger has changed the number of threads for the parallel region at source file
    location ident.  This way, specific parallel regions' number of threads can be changed
    at the debugger's request.
  */

runtime/src/kmp_dispatch.cpp

@@ -25,6 +25,12 @@
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
+// Need to raise Win version from XP to Vista here for support of InterlockedExchange64
+#if defined(_WIN32_WINNT) && defined(_M_IX86)
+#undef _WIN32_WINNT
+#define _WIN32_WINNT 0x0502
+#endif
+
 #include "kmp.h"
 #include "kmp_i18n.h"
 #include "kmp_itt.h"
@@ -71,7 +77,7 @@ struct i_maxmin< unsigned long long > {
 };
 //-------------------------------------------------------------------------
 
-#ifdef KMP_STATIC_STEAL_ENABLED
+#if KMP_STATIC_STEAL_ENABLED
 
     // replaces dispatch_private_info{32,64} structures and dispatch_private_info{32,64}_t types
     template< typename T >
@@ -163,7 +169,7 @@ struct dispatch_shared_infoXX_template {
     volatile UT     iteration;
     volatile UT     num_done;
     volatile UT     ordered_iteration;
-    UT   ordered_dummy[KMP_MAX_ORDERED-1]; // to retain the structure size making ordered_iteration scalar
+    UT   ordered_dummy[KMP_MAX_ORDERED-3]; // to retain the structure size making ordered_iteration scalar
 };
 
 // replaces dispatch_shared_info structure and dispatch_shared_info_t type
@@ -175,6 +181,17 @@ struct dispatch_shared_info_template {
         dispatch_shared_info64_t               s64;
     } u;
     volatile kmp_uint32     buffer_index;
+#if OMP_45_ENABLED
+    volatile kmp_int32      doacross_buf_idx;  // teamwise index
+    kmp_uint32             *doacross_flags;    // array of iteration flags (0/1)
+    kmp_int32               doacross_num_done; // count finished threads
+#endif
+#if KMP_USE_HWLOC
+    // When linking with libhwloc, the ORDERED EPCC test slowsdown on big
+    // machines (> 48 cores). Performance analysis showed that a cache thrash
+    // was occurring and this padding helps alleviate the problem.
+    char padding[64];
+#endif
 };
 
 /* ------------------------------------------------------------------------ */
@@ -185,7 +202,7 @@ struct dispatch_shared_info_template {
 // test_then_add template (general template should NOT be used)
 template< typename T >
 static __forceinline T
-test_then_add( volatile T *p, T d ) { KMP_ASSERT(0); };
+test_then_add( volatile T *p, T d );
 
 template<>
 __forceinline kmp_int32
@@ -208,7 +225,7 @@ test_then_add< kmp_int64 >( volatile kmp_int64 *p, kmp_int64 d )
 // test_then_inc_acq template (general template should NOT be used)
 template< typename T >
 static __forceinline T
-test_then_inc_acq( volatile T *p ) { KMP_ASSERT(0); };
+test_then_inc_acq( volatile T *p );
 
 template<>
 __forceinline kmp_int32
@@ -231,7 +248,7 @@ test_then_inc_acq< kmp_int64 >( volatile kmp_int64 *p )
 // test_then_inc template (general template should NOT be used)
 template< typename T >
 static __forceinline T
-test_then_inc( volatile T *p ) { KMP_ASSERT(0); };
+test_then_inc( volatile T *p );
 
 template<>
 __forceinline kmp_int32
@@ -254,7 +271,7 @@ test_then_inc< kmp_int64 >( volatile kmp_int64 *p )
 // compare_and_swap template (general template should NOT be used)
 template< typename T >
 static __forceinline kmp_int32
-compare_and_swap( volatile T *p, T c, T s ) { KMP_ASSERT(0); };
+compare_and_swap( volatile T *p, T c, T s );
 
 template<>
 __forceinline kmp_int32
@@ -651,11 +668,19 @@ __kmp_dispatch_init(
 
         /* What happens when number of threads changes, need to resize buffer? */
         pr = reinterpret_cast< dispatch_private_info_template< T >  * >
-            ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
+            ( &th -> th.th_dispatch -> th_disp_buffer[ my_buffer_index % __kmp_dispatch_num_buffers ] );
         sh = reinterpret_cast< dispatch_shared_info_template< UT > volatile * >
-            ( &team -> t.t_disp_buffer[ my_buffer_index % KMP_MAX_DISP_BUF ] );
+            ( &team -> t.t_disp_buffer[ my_buffer_index % __kmp_dispatch_num_buffers ] );
     }
 
+    #if  ( KMP_STATIC_STEAL_ENABLED )
+    if ( SCHEDULE_HAS_NONMONOTONIC(schedule) )
+        // AC: we now have only one implementation of stealing, so use it
+        schedule = kmp_sch_static_steal;
+    else
+    #endif
+        schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
+
     /* Pick up the nomerge/ordered bits from the scheduling type */
     if ( (schedule >= kmp_nm_lower) && (schedule < kmp_nm_upper) ) {
         pr->nomerge = TRUE;
@@ -725,7 +750,7 @@ __kmp_dispatch_init(
         }
 
         /* guided analytical not safe for too many threads */
-        if ( team->t.t_nproc > 1<<20 && schedule == kmp_sch_guided_analytical_chunked ) {
+        if ( schedule == kmp_sch_guided_analytical_chunked && th->th.th_team_nproc > 1<<20 ) {
             schedule = kmp_sch_guided_iterative_chunked;
             KMP_WARNING( DispatchManyThreads );
         }
@@ -744,24 +769,29 @@ __kmp_dispatch_init(
             );
         }
     }
-
-    tc = ( ub - lb + st );
-    if ( st != 1 ) {
-        if ( st < 0 ) {
-            if ( lb < ub ) {
-                tc = 0;            // zero-trip
-            } else {   // lb >= ub
-                tc = (ST)tc / st;  // convert to signed division
-            }
-        } else {       // st > 0
-            if ( ub < lb ) {
-                tc = 0;            // zero-trip
-            } else {   // lb >= ub
-                tc /= st;
-            }
+    // compute trip count
+    if ( st == 1 ) {   // most common case
+        if ( ub >= lb ) {
+            tc = ub - lb + 1;
+        } else {   // ub < lb
+            tc = 0;            // zero-trip
+        }
+    } else if ( st < 0 ) {
+        if ( lb >= ub ) {
+            // AC: cast to unsigned is needed for loops like (i=2B; i>-2B; i-=1B),
+            //     where the division needs to be unsigned regardless of the result type
+            tc = (UT)(lb - ub) / (-st) + 1;
+        } else {   // lb < ub
+            tc = 0;            // zero-trip
+        }
+    } else {       // st > 0
+        if ( ub >= lb ) {
+            // AC: cast to unsigned is needed for loops like (i=-2B; i<2B; i+=1B),
+            //     where the division needs to be unsigned regardless of the result type
+            tc = (UT)(ub - lb) / st + 1;
+        } else {   // ub < lb
+            tc = 0;            // zero-trip
         }
-    } else if ( ub < lb ) {        // st == 1
-        tc = 0;                    // zero-trip
     }
 
     // Any half-decent optimizer will remove this test when the blocks are empty since the macros expand to nothing
@@ -815,10 +845,10 @@ __kmp_dispatch_init(
     }
 
     switch ( schedule ) {
-    #if  ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 )
+    #if  ( KMP_STATIC_STEAL_ENABLED )
     case kmp_sch_static_steal:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             T ntc, init;
 
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_steal case\n", gtid ) );
@@ -837,8 +867,19 @@ __kmp_dispatch_init(
 
                 pr->u.p.parm2 = lb;
                 //pr->pfields.parm3 = 0; // it's not used in static_steal
-                pr->u.p.parm4 = id;
+                pr->u.p.parm4 = (id + 1) % nproc; // remember neighbour tid
                 pr->u.p.st = st;
+                if ( ___kmp_size_type > 4 ) {
+                    // AC: TODO: check if 16-byte CAS available and use it to
+                    // improve performance (probably wait for explicit request
+                    // before spending time on this).
+                    // For now use dynamically allocated per-thread lock,
+                    // free memory in __kmp_dispatch_next when status==0.
+                    KMP_DEBUG_ASSERT(th->th.th_dispatch->th_steal_lock == NULL);
+                    th->th.th_dispatch->th_steal_lock =
+                        (kmp_lock_t*)__kmp_allocate(sizeof(kmp_lock_t));
+                    __kmp_init_lock(th->th.th_dispatch->th_steal_lock);
+                }
                 break;
             } else {
                 KD_TRACE(100, ("__kmp_dispatch_init: T#%d falling-through to kmp_sch_static_balanced\n",
@@ -851,7 +892,7 @@ __kmp_dispatch_init(
     #endif
     case kmp_sch_static_balanced:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             T init, limit;
 
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_static_balanced case\n",
@@ -915,7 +956,7 @@ __kmp_dispatch_init(
         } // case
     case kmp_sch_guided_iterative_chunked :
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_guided_iterative_chunked case\n",gtid));
 
             if ( nproc > 1 ) {
@@ -938,7 +979,7 @@ __kmp_dispatch_init(
         break;
     case kmp_sch_guided_analytical_chunked:
         {
-            T nproc = team->t.t_nproc;
+            T nproc = th->th.th_team_nproc;
             KD_TRACE(100, ("__kmp_dispatch_init: T#%d kmp_sch_guided_analytical_chunked case\n", gtid));
 
             if ( nproc > 1 ) {
@@ -1056,8 +1097,8 @@ __kmp_dispatch_init(
         break;
     case kmp_sch_static_greedy:
         KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_static_greedy case\n",gtid));
-            pr->u.p.parm1 = ( team -> t.t_nproc > 1 ) ?
-                ( tc + team->t.t_nproc - 1 ) / team->t.t_nproc :
+            pr->u.p.parm1 = ( th->th.th_team_nproc > 1 ) ?
+                ( tc + th->th.th_team_nproc - 1 ) / th->th.th_team_nproc :
                 tc;
         break;
     case kmp_sch_static_chunked :
@@ -1077,7 +1118,7 @@ __kmp_dispatch_init(
             parm1 = chunk;
 
             /* F : size of the first cycle */
-            parm2 = ( tc / (2 * team->t.t_nproc) );
+            parm2 = ( tc / (2 * th->th.th_team_nproc) );
 
             if ( parm2 < 1 ) {
                 parm2 = 1;
@@ -1198,7 +1239,6 @@ __kmp_dispatch_init(
     }
     #endif
     #if ( KMP_STATIC_STEAL_ENABLED )
-    if ( ___kmp_size_type < 8 ) {
       // It cannot be guaranteed that after execution of a loop with some other schedule kind
       // all the parm3 variables will contain the same value.
       // Even if all parm3 will be the same, it still exists a bad case like using 0 and 1
@@ -1210,8 +1250,7 @@ __kmp_dispatch_init(
         volatile T * p = &pr->u.p.static_steal_counter;
         *p = *p + 1;
       }
-    }
-    #endif // ( KMP_STATIC_STEAL_ENABLED && USE_STEALING )
+    #endif // ( KMP_STATIC_STEAL_ENABLED )
 
 #if OMPT_SUPPORT && OMPT_TRACE
     if (ompt_enabled &&
@@ -1399,14 +1438,14 @@ __kmp_dispatch_next(
     typedef typename traits_t< T >::unsigned_t  UT;
     typedef typename traits_t< T >::signed_t    ST;
     typedef typename traits_t< T >::floating_t  DBL;
-#if ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 )
+#if ( KMP_STATIC_STEAL_ENABLED )
     static const int ___kmp_size_type = sizeof( UT );
 #endif
 
     // This is potentially slightly misleading, schedule(runtime) will appear here even if the actual runtme schedule
     // is static. (Which points out a disadavantage of schedule(runtime): even when static scheduling is used it costs
     // more than a compile time choice to use static scheduling would.)
-    KMP_TIME_BLOCK(FOR_dynamic_scheduling);
+    KMP_TIME_PARTITIONED_BLOCK(FOR_dynamic_scheduling);
 
     int                                   status;
     dispatch_private_info_template< T > * pr;
@@ -1558,21 +1597,97 @@ __kmp_dispatch_next(
             status = 0;
         } else {
             switch (pr->schedule) {
-            #if ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 )
+            #if ( KMP_STATIC_STEAL_ENABLED )
             case kmp_sch_static_steal:
                 {
                     T chunk = pr->u.p.parm1;
+                    int nproc = th->th.th_team_nproc;
 
                     KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_static_steal case\n", gtid) );
 
                     trip = pr->u.p.tc - 1;
 
                     if ( ___kmp_size_type > 4 ) {
-                        // Other threads do not look into the data of this thread,
-                        //  so it's not necessary to make volatile casting.
-                        init   = ( pr->u.p.count )++;
-                        status = ( init < (UT)pr->u.p.ub );
+                        // use lock for 8-byte and CAS for 4-byte induction
+                        // variable. TODO (optional): check and use 16-byte CAS
+                        kmp_lock_t * lck = th->th.th_dispatch->th_steal_lock;
+                        KMP_DEBUG_ASSERT(lck != NULL);
+                        if( pr->u.p.count < (UT)pr->u.p.ub ) {
+                            __kmp_acquire_lock(lck, gtid);
+                            // try to get own chunk of iterations
+                            init   = ( pr->u.p.count )++;
+                            status = ( init < (UT)pr->u.p.ub );
+                            __kmp_release_lock(lck, gtid);
+                        } else {
+                            status = 0; // no own chunks
+                        }
+                        if( !status ) { // try to steal
+                            kmp_info_t   **other_threads = team->t.t_threads;
+                            int          while_limit = nproc; // nproc attempts to find a victim
+                            int          while_index = 0;
+                            // TODO: algorithm of searching for a victim
+                            // should be cleaned up and measured
+                            while ( ( !status ) && ( while_limit != ++while_index ) ) {
+                                T remaining;
+                                T victimIdx    = pr->u.p.parm4;
+                                T oldVictimIdx = victimIdx ? victimIdx - 1 : nproc - 1;
+                                dispatch_private_info_template< T > * victim =
+                                    reinterpret_cast< dispatch_private_info_template< T >* >
+                                    (other_threads[victimIdx]->th.th_dispatch->th_dispatch_pr_current);
+                                while( ( victim == NULL || victim == pr ||
+                                    ( *(volatile T*)&victim->u.p.static_steal_counter !=
+                                    *(volatile T*)&pr->u.p.static_steal_counter ) ) &&
+                                    oldVictimIdx != victimIdx )
+                                {
+                                    victimIdx = (victimIdx + 1) % nproc;
+                                    victim = reinterpret_cast< dispatch_private_info_template< T >* >
+                                        (other_threads[victimIdx]->th.th_dispatch->th_dispatch_pr_current);
+                                };
+                                if( !victim ||
+                                    ( *(volatile T *)&victim->u.p.static_steal_counter !=
+                                    *(volatile T *)&pr->u.p.static_steal_counter ) )
+                                {
+                                    continue; // try once more (nproc attempts in total)
+                                    // no victim is ready yet to participate in stealing
+                                    // because all victims are still in kmp_init_dispatch
+                                }
+                                if( victim->u.p.count + 2 > (UT)victim->u.p.ub ) {
+                                    pr->u.p.parm4 = (victimIdx + 1) % nproc; // shift start tid
+                                    continue; // not enough chunks to steal, goto next victim
+                                }
+
+                                lck = other_threads[victimIdx]->th.th_dispatch->th_steal_lock;
+                                KMP_ASSERT(lck != NULL);
+                                __kmp_acquire_lock(lck, gtid);
+                                limit = victim->u.p.ub; // keep initial ub
+                                if( victim->u.p.count >= limit ||
+                                    (remaining = limit - victim->u.p.count) < 2 )
+                                {
+                                    __kmp_release_lock(lck, gtid);
+                                    pr->u.p.parm4 = (victimIdx + 1) % nproc; // next victim
+                                    continue; // not enough chunks to steal
+                                }
+                                // stealing succeded, reduce victim's ub by 1/4 of undone chunks or by 1
+                                if( remaining > 3 ) {
+                                    init = ( victim->u.p.ub -= (remaining>>2) ); // steal 1/4 of remaining
+                                } else {
+                                    init = ( victim->u.p.ub -= 1 ); // steal 1 chunk of 2 or 3 remaining
+                                }
+                                __kmp_release_lock(lck, gtid);
+
+                                KMP_DEBUG_ASSERT(init + 1 <= limit);
+                                pr->u.p.parm4 = victimIdx; // remember victim to steal from
+                                status = 1;
+                                while_index = 0;
+                                // now update own count and ub with stolen range but init chunk
+                                __kmp_acquire_lock(th->th.th_dispatch->th_steal_lock, gtid);
+                                pr->u.p.count = init + 1;
+                                pr->u.p.ub = limit;
+                                __kmp_release_lock(th->th.th_dispatch->th_steal_lock, gtid);
+                            } // while (search for victim)
+                        } // if (try to find victim and steal)
                     } else {
+                        // 4-byte induction variable, use 8-byte CAS for pair (count, ub)
                         typedef union {
                             struct {
                                 UT count;
@@ -1581,7 +1696,6 @@ __kmp_dispatch_next(
                             kmp_int64 b;
                         } union_i4;
                         // All operations on 'count' or 'ub' must be combined atomically together.
-                        // stealing implemented only for 4-byte indexes
                         {
                             union_i4 vold, vnew;
                             vold.b = *( volatile kmp_int64 * )(&pr->u.p.count);
@@ -1603,86 +1717,77 @@ __kmp_dispatch_next(
 
                         if( !status ) {
                             kmp_info_t   **other_threads = team->t.t_threads;
-                            int          while_limit = 10;
+                            int          while_limit = nproc; // nproc attempts to find a victim
                             int          while_index = 0;
 
                             // TODO: algorithm of searching for a victim
                             // should be cleaned up and measured
                             while ( ( !status ) && ( while_limit != ++while_index ) ) {
                                 union_i4  vold, vnew;
-                                kmp_int32 remaining; // kmp_int32 because KMP_I4 only
+                                kmp_int32 remaining;
                                 T         victimIdx    = pr->u.p.parm4;
-                                T         oldVictimIdx = victimIdx;
-                                dispatch_private_info_template< T > * victim;
-
-                                do {
-                                    if( !victimIdx ) {
-                                        victimIdx = team->t.t_nproc - 1;
-                                    } else {
-                                        --victimIdx;
-                                    }
+                                T oldVictimIdx = victimIdx ? victimIdx - 1 : nproc - 1;
+                                dispatch_private_info_template< T > * victim =
+                                    reinterpret_cast< dispatch_private_info_template< T >* >
+                                    (other_threads[victimIdx]->th.th_dispatch->th_dispatch_pr_current);
+                                while( (victim == NULL || victim == pr ||
+                                    (*(volatile T*)&victim->u.p.static_steal_counter !=
+                                    *(volatile T*)&pr->u.p.static_steal_counter)) &&
+                                    oldVictimIdx != victimIdx )
+                                {
+                                    victimIdx = (victimIdx + 1) % nproc;
                                     victim = reinterpret_cast< dispatch_private_info_template< T >* >
                                         ( other_threads[victimIdx]->th.th_dispatch->th_dispatch_pr_current );
-                                } while ( (victim == NULL || victim == pr) && oldVictimIdx != victimIdx );
-                                // TODO: think about a proper place of this test
-                                if ( ( !victim ) ||
-                                   ( (*( volatile T * )&victim->u.p.static_steal_counter) !=
-                                     (*( volatile T * )&pr->u.p.static_steal_counter) ) ) {
-                                    // TODO: delay would be nice
-                                    continue;
-                                    // the victim is not ready yet to participate in stealing
-                                    // because the victim is still in kmp_init_dispatch
+                                };
+                                if( !victim ||
+                                    ( *(volatile T *)&victim->u.p.static_steal_counter !=
+                                    *(volatile T *)&pr->u.p.static_steal_counter ) )
+                                {
+                                    continue; // try once more (nproc attempts in total)
+                                    // no victim is ready yet to participate in stealing
+                                    // because all victims are still in kmp_init_dispatch
                                 }
-                                if ( oldVictimIdx == victimIdx ) {
-                                    break;
-                                }
-                                pr->u.p.parm4 = victimIdx;
-
-                                while( 1 ) {
+                                pr->u.p.parm4 = victimIdx; // new victim found
+                                while( 1 ) { // CAS loop if victim has enough chunks to steal
                                     vold.b = *( volatile kmp_int64 * )( &victim->u.p.count );
                                     vnew = vold;
 
                                     KMP_DEBUG_ASSERT( (vnew.p.ub - 1) * (UT)chunk <= trip );
-                                    if ( vnew.p.count >= (UT)vnew.p.ub || (remaining = vnew.p.ub - vnew.p.count) < 4 ) {
-                                        break;
+                                    if ( vnew.p.count >= (UT)vnew.p.ub ||
+                                        (remaining = vnew.p.ub - vnew.p.count) < 2 )
+                                    {
+                                        pr->u.p.parm4 = (victimIdx + 1) % nproc; // shift start victim id
+                                        break; // not enough chunks to steal, goto next victim
+                                    }
+                                    if( remaining > 3 ) {
+                                        vnew.p.ub -= (remaining>>2); // try to steal 1/4 of remaining
+                                    } else {
+                                        vnew.p.ub -= 1; // steal 1 chunk of 2 or 3 remaining
                                     }
-                                    vnew.p.ub -= (remaining >> 2);
                                     KMP_DEBUG_ASSERT((vnew.p.ub - 1) * (UT)chunk <= trip);
-                                    #pragma warning( push )
-                                    // disable warning on pointless comparison of unsigned with 0
-                                    #pragma warning( disable: 186 )
-                                        KMP_DEBUG_ASSERT(vnew.p.ub >= 0);
-                                    #pragma warning( pop )
                                     // TODO: Should this be acquire or release?
                                     if ( KMP_COMPARE_AND_STORE_ACQ64(
                                             ( volatile kmp_int64 * )&victim->u.p.count,
                                             *VOLATILE_CAST(kmp_int64 *)&vold.b,
                                             *VOLATILE_CAST(kmp_int64 *)&vnew.b ) ) {
+                                        // stealing succedded
                                         status = 1;
                                         while_index = 0;
                                         // now update own count and ub
+                                        init = vnew.p.ub;
+                                        vold.p.count = init + 1;
                                         #if KMP_ARCH_X86
-                                        // stealing executed on non-KMP_ARCH_X86 only
-                                            // Atomic 64-bit write on ia32 is
-                                            // unavailable, so we do this in steps.
-                                            //     This code is not tested.
-                                            init = vold.p.count;
-                                            pr->u.p.ub = 0;
-                                            pr->u.p.count = init + 1;
-                                            pr->u.p.ub = vnew.p.count;
+                                        KMP_XCHG_FIXED64(( volatile kmp_int64 * )(&pr->u.p.count), vold.b);
                                         #else
-                                            init = vnew.p.ub;
-                                            vold.p.count = init + 1;
-                                            // TODO: is it safe and enough?
-                                            *( volatile kmp_int64 * )(&pr->u.p.count) = vold.b;
-                                        #endif // KMP_ARCH_X86
+                                        *( volatile kmp_int64 * )(&pr->u.p.count) = vold.b;
+                                        #endif
                                         break;
-                                    } // if
-                                KMP_CPU_PAUSE();
-                                } // while (1)
-                            } // while
-                        } // if
-                    } // if
+                                    } // if (check CAS result)
+                                    KMP_CPU_PAUSE(); // CAS failed, repeate attempt
+                                } // while (try to steal from particular victim)
+                            } // while (search for victim)
+                        } // if (try to find victim and steal)
+                    } // if (4-byte induction variable)
                     if ( !status ) {
                         *p_lb = 0;
                         *p_ub = 0;
@@ -1724,7 +1829,7 @@ __kmp_dispatch_next(
                     } // if
                     break;
                 } // case
-            #endif // ( KMP_STATIC_STEAL_ENABLED && KMP_ARCH_X86_64 )
+            #endif // ( KMP_STATIC_STEAL_ENABLED )
             case kmp_sch_static_balanced:
                 {
                     KD_TRACE(100, ("__kmp_dispatch_next: T#%d kmp_sch_static_balanced case\n", gtid) );
@@ -1775,7 +1880,7 @@ __kmp_dispatch_next(
 
                         if ( p_st != NULL ) *p_st = incr;
 
-                        pr->u.p.count += team->t.t_nproc;
+                        pr->u.p.count += th->th.th_team_nproc;
 
                         if ( incr == 1 ) {
                             *p_lb = start + init;
@@ -1945,8 +2050,8 @@ __kmp_dispatch_next(
 
                     trip  = pr->u.p.tc;
 
-                    KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
-                    KMP_DEBUG_ASSERT((2UL * chunkspec + 1) * (UT)team->t.t_nproc < trip);
+                    KMP_DEBUG_ASSERT(th->th.th_team_nproc > 1);
+                    KMP_DEBUG_ASSERT((2UL * chunkspec + 1) * (UT)th->th.th_team_nproc < trip);
 
                     while(1) { /* this while loop is a safeguard against unexpected zero chunk sizes */
                         chunkIdx = test_then_inc_acq< ST >((volatile ST *) & sh->u.s.iteration );
@@ -2117,7 +2222,21 @@ __kmp_dispatch_next(
             }
             #endif
 
-            if ( (ST)num_done == team->t.t_nproc-1 ) {
+            if ( (ST)num_done == th->th.th_team_nproc - 1 ) {
+                #if ( KMP_STATIC_STEAL_ENABLED )
+                if( pr->schedule == kmp_sch_static_steal && ___kmp_size_type > 4 ) {
+                    int i;
+                    kmp_info_t **other_threads = team->t.t_threads;
+                    // loop complete, safe to destroy locks used for stealing
+                    for( i = 0; i < th->th.th_team_nproc; ++i ) {
+                        kmp_lock_t * lck = other_threads[i]->th.th_dispatch->th_steal_lock;
+                        KMP_ASSERT(lck != NULL);
+                        __kmp_destroy_lock( lck );
+                        __kmp_free( lck );
+                        other_threads[i]->th.th_dispatch->th_steal_lock = NULL;
+                    }
+                }
+                #endif
                 /* NOTE: release this buffer to be reused */
 
                 KMP_MB();       /* Flush all pending memory write invalidates.  */
@@ -2132,7 +2251,7 @@ __kmp_dispatch_next(
 
                 KMP_MB();       /* Flush all pending memory write invalidates.  */
 
-                sh -> buffer_index += KMP_MAX_DISP_BUF;
+                sh -> buffer_index += __kmp_dispatch_num_buffers;
                 KD_TRACE(100, ("__kmp_dispatch_next: T#%d change buffer_index:%d\n",
                                 gtid, sh->buffer_index) );
 
@@ -2242,8 +2361,11 @@ __kmp_dist_get_bounds(
         trip_count = *pupper - *plower + 1;
     } else if(incr == -1) {
         trip_count = *plower - *pupper + 1;
+    } else if ( incr > 0 ) {
+        // upper-lower can exceed the limit of signed type
+        trip_count = (UT)(*pupper - *plower) / incr + 1;
     } else {
-        trip_count = (ST)(*pupper - *plower) / incr + 1; // cast to signed to cover incr<0 case
+        trip_count = (UT)(*plower - *pupper) / ( -incr ) + 1;
     }
 
     if( trip_count <= nteams ) {
@@ -2494,7 +2616,7 @@ __kmpc_dispatch_fini_8u( ident_t *loc, kmp_int32 gtid )
 /*! @} */
 
 //-----------------------------------------------------------------------------------------
-//Non-template routines from kmp_dispatch.c used in other sources
+//Non-template routines from kmp_dispatch.cpp used in other sources
 
 kmp_uint32 __kmp_eq_4( kmp_uint32 value, kmp_uint32 checker) {
     return value == checker;
@@ -2549,6 +2671,32 @@ __kmp_wait_yield_4(volatile kmp_uint32 * spinner,
     return r;
 }
 
+void
+__kmp_wait_yield_4_ptr(void *spinner,
+                   kmp_uint32 checker,
+                   kmp_uint32 (*pred)( void *, kmp_uint32 ),
+                   void        *obj    // Higher-level synchronization object, or NULL.
+                   )
+{
+    // note: we may not belong to a team at this point
+    register void                *spin          = spinner;
+    register kmp_uint32           check         = checker;
+    register kmp_uint32           spins;
+    register kmp_uint32 (*f) ( void *, kmp_uint32 ) = pred;
+
+    KMP_FSYNC_SPIN_INIT( obj, spin );
+    KMP_INIT_YIELD( spins );
+    // main wait spin loop
+    while ( !f( spin, check ) ) {
+        KMP_FSYNC_SPIN_PREPARE( obj );
+        /* if we have waited a bit, or are oversubscribed, yield */
+        /* pause is in the following code */
+        KMP_YIELD( TCR_4( __kmp_nth ) > __kmp_avail_proc );
+        KMP_YIELD_SPIN( spins );
+    }
+    KMP_FSYNC_SPIN_ACQUIRED( obj );
+}
+
 } // extern "C"
 
 #ifdef KMP_GOMP_COMPAT

runtime/src/kmp_environment.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_environment.c -- Handle environment variables OS-independently.
+ * kmp_environment.cpp -- Handle environment variables OS-independently.
  */
 
 
@@ -23,17 +23,17 @@
     ------------------------------------------------------------------------------------------------
     On Windows* OS, there are two environments (at least, see below):
 
-        1. Environment maintained by Windows* OS on IA-32 architecture. 
+        1. Environment maintained by Windows* OS on IA-32 architecture.
 	   Accessible through GetEnvironmentVariable(),
            SetEnvironmentVariable(), and GetEnvironmentStrings().
 
         2. Environment maintained by C RTL. Accessible through getenv(), putenv().
 
-    putenv() function updates both C and Windows* OS on IA-32 architecture. getenv() function 
-    search for variables in C RTL environment only. Windows* OS on IA-32 architecture functions work *only* 
+    putenv() function updates both C and Windows* OS on IA-32 architecture. getenv() function
+    search for variables in C RTL environment only. Windows* OS on IA-32 architecture functions work *only*
     with Windows* OS on IA-32 architecture.
 
-    Windows* OS on IA-32 architecture maintained by OS, so there is always only one Windows* OS on 
+    Windows* OS on IA-32 architecture maintained by OS, so there is always only one Windows* OS on
     IA-32 architecture per process. Changes in Windows* OS on IA-32 architecture are process-visible.
 
     C environment maintained by C RTL. Multiple copies of C RTL may be present in the process, and
@@ -42,11 +42,11 @@
     Thus, proper way to work with environment on Windows* OS is:
 
         1. Set variables with putenv() function -- both C and Windows* OS on
-	   IA-32 architecture are being updated. Windows* OS on 
+	   IA-32 architecture are being updated. Windows* OS on
 	   IA-32 architecture may be considered as primary target,
 	   while updating C RTL environment is a free bonus.
 
-        2. Get variables with GetEnvironmentVariable() -- getenv() does not 
+        2. Get variables with GetEnvironmentVariable() -- getenv() does not
 	   search Windows* OS on IA-32 architecture, and can not see variables
 	   set with SetEnvironmentVariable().
 
@@ -214,7 +214,7 @@ __kmp_env_set( char const * name, char const * value, int overwrite ) {
             // Dead code. I tried to put too many variables into Linux* OS
             // environment on IA-32 architecture. When application consumes
             // more than ~2.5 GB of memory, entire system feels bad. Sometimes
-            // application is killed (by OS?), sometimes system stops 
+            // application is killed (by OS?), sometimes system stops
             // responding... But this error message never appears. --ln
             __kmp_msg(
                 kmp_ms_fatal,
@@ -566,11 +566,10 @@ __kmp_env_blk_free(
 ) {
 
     KMP_INTERNAL_FREE( (void *) block->vars );
-    KMP_INTERNAL_FREE( (void *) block->bulk );
+    __kmp_str_free(&(block->bulk));
 
     block->count = 0;
     block->vars  = NULL;
-    block->bulk  = NULL;
 
 } // __kmp_env_blk_free
 

runtime/src/kmp_error.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_error.c -- KPTS functions for error checking at runtime
+ * kmp_error.cpp -- KPTS functions for error checking at runtime
  */
 
 

runtime/src/kmp_ftn_cdecl.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_ftn_cdecl.c -- Fortran __cdecl linkage support for OpenMP.
+ * kmp_ftn_cdecl.cpp -- Fortran __cdecl linkage support for OpenMP.
  */
 
 
@@ -14,6 +14,7 @@
 
 
 #include "kmp.h"
+#include "kmp_affinity.h"
 
 #if KMP_OS_WINDOWS
 #   if defined  KMP_WIN_CDECL ||  !defined KMP_DYNAMIC_LIB

runtime/src/kmp_ftn_entry.h

@@ -215,6 +215,19 @@ FTN_GET_LIBRARY (void)
     #endif
 }
 
+void FTN_STDCALL
+FTN_SET_DISP_NUM_BUFFERS( int KMP_DEREF arg )
+{
+    #ifdef KMP_STUB
+        ; // empty routine
+    #else
+        // ignore after initialization because some teams have already
+        // allocated dispatch buffers
+        if( __kmp_init_serial == 0 && (KMP_DEREF arg) > 0 )
+            __kmp_dispatch_num_buffers = KMP_DEREF arg;
+    #endif
+}
+
 int FTN_STDCALL
 FTN_SET_AFFINITY( void **mask )
 {
@@ -253,16 +266,7 @@ FTN_GET_AFFINITY_MAX_PROC( void )
         if ( ! TCR_4(__kmp_init_middle) ) {
             __kmp_middle_initialize();
         }
-        if ( ! ( KMP_AFFINITY_CAPABLE() ) ) {
-            return 0;
-        }
-
-    #if KMP_GROUP_AFFINITY && !KMP_USE_HWLOC
-        if ( __kmp_num_proc_groups > 1 ) {
-            return (int)KMP_CPU_SETSIZE;
-        }
-    #endif /* KMP_GROUP_AFFINITY */
-        return __kmp_xproc;
+        return __kmp_aux_get_affinity_max_proc();
     #endif
 }
 
@@ -275,15 +279,13 @@ FTN_CREATE_AFFINITY_MASK( void **mask )
         //
         // We really only NEED serial initialization here.
         //
+        kmp_affin_mask_t* mask_internals;
         if ( ! TCR_4(__kmp_init_middle) ) {
             __kmp_middle_initialize();
         }
-    # if KMP_USE_HWLOC
-        *mask = (hwloc_cpuset_t)hwloc_bitmap_alloc();
-    # else
-        *mask = kmpc_malloc( __kmp_affin_mask_size );
-    # endif
-        KMP_CPU_ZERO( (kmp_affin_mask_t *)(*mask) );
+        mask_internals = __kmp_affinity_dispatch->allocate_mask();
+        KMP_CPU_ZERO( mask_internals );
+        *mask = mask_internals;
     #endif
 }
 
@@ -296,6 +298,7 @@ FTN_DESTROY_AFFINITY_MASK( void **mask )
         //
         // We really only NEED serial initialization here.
         //
+        kmp_affin_mask_t* mask_internals;
         if ( ! TCR_4(__kmp_init_middle) ) {
             __kmp_middle_initialize();
         }
@@ -304,11 +307,8 @@ FTN_DESTROY_AFFINITY_MASK( void **mask )
 	        KMP_FATAL( AffinityInvalidMask, "kmp_destroy_affinity_mask" );
 	    }
         }
-    # if KMP_USE_HWLOC
-        hwloc_bitmap_free((hwloc_cpuset_t)(*mask));
-    # else
-        kmpc_free( *mask );
-    # endif
+        mask_internals = (kmp_affin_mask_t*)(*mask);
+        __kmp_affinity_dispatch->deallocate_mask(mask_internals);
         *mask = NULL;
     #endif
 }
@@ -676,6 +676,155 @@ xexpand(FTN_GET_PROC_BIND)( void )
     #endif
 }
 
+#if OMP_45_ENABLED
+int FTN_STDCALL
+FTN_GET_NUM_PLACES( void )
+{
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+        return 0;
+    #else
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return 0;
+        return __kmp_affinity_num_masks;
+    #endif
+}
+
+int FTN_STDCALL
+FTN_GET_PLACE_NUM_PROCS( int place_num )
+{
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+        return 0;
+    #else
+        int i;
+        int retval = 0;
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return 0;
+        if ( place_num < 0 || place_num >= (int)__kmp_affinity_num_masks )
+            return 0;
+        kmp_affin_mask_t *mask = KMP_CPU_INDEX(__kmp_affinity_masks, place_num);
+        KMP_CPU_SET_ITERATE(i, mask) {
+            if ((! KMP_CPU_ISSET(i, __kmp_affin_fullMask)) ||
+              (!KMP_CPU_ISSET(i, mask))) {
+                continue;
+            }
+            ++retval;
+        }
+        return retval;
+    #endif
+}
+
+void FTN_STDCALL
+FTN_GET_PLACE_PROC_IDS( int place_num, int *ids )
+{
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+    // Nothing.
+    #else
+        int i,j;
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return;
+        if ( place_num < 0 || place_num >= (int)__kmp_affinity_num_masks )
+            return;
+        kmp_affin_mask_t *mask = KMP_CPU_INDEX(__kmp_affinity_masks, place_num);
+        j = 0;
+        KMP_CPU_SET_ITERATE(i, mask) {
+            if ((! KMP_CPU_ISSET(i, __kmp_affin_fullMask)) ||
+              (!KMP_CPU_ISSET(i, mask))) {
+                continue;
+            }
+            ids[j++] = i;
+        }
+    #endif
+}
+
+int FTN_STDCALL
+FTN_GET_PLACE_NUM( void )
+{
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+        return -1;
+    #else
+        int gtid;
+        kmp_info_t *thread;
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return -1;
+        gtid = __kmp_entry_gtid();
+        thread = __kmp_thread_from_gtid(gtid);
+        if ( thread->th.th_current_place < 0 )
+            return -1;
+        return thread->th.th_current_place;
+    #endif
+}
+
+int FTN_STDCALL
+FTN_GET_PARTITION_NUM_PLACES( void )
+{
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+        return 0;
+    #else
+        int gtid, num_places, first_place, last_place;
+        kmp_info_t *thread;
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return 0;
+        gtid = __kmp_entry_gtid();
+        thread = __kmp_thread_from_gtid(gtid);
+        first_place = thread->th.th_first_place;
+        last_place = thread->th.th_last_place;
+        if ( first_place < 0 || last_place < 0 )
+            return 0;
+        if ( first_place <= last_place )
+            num_places = last_place - first_place + 1;
+        else
+            num_places = __kmp_affinity_num_masks - first_place + last_place + 1;
+        return num_places;
+    #endif
+}
+
+void FTN_STDCALL
+FTN_GET_PARTITION_PLACE_NUMS( int *place_nums ) {
+    #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
+    // Nothing.
+    #else
+        int i, gtid, place_num, first_place, last_place, start, end;
+        kmp_info_t *thread;
+        if ( ! TCR_4(__kmp_init_middle) ) {
+            __kmp_middle_initialize();
+        }
+        if (!KMP_AFFINITY_CAPABLE())
+            return;
+        gtid = __kmp_entry_gtid();
+        thread = __kmp_thread_from_gtid(gtid);
+        first_place = thread->th.th_first_place;
+        last_place = thread->th.th_last_place;
+        if ( first_place < 0 || last_place < 0 )
+            return;
+        if ( first_place <= last_place ) {
+            start = first_place;
+            end = last_place;
+        } else {
+            start = last_place;
+            end = first_place;
+        }
+        for (i = 0, place_num = start; place_num <= end; ++place_num, ++i) {
+            place_nums[i] = place_num;
+        }
+    #endif
+}
+#endif
+
 int FTN_STDCALL
 xexpand(FTN_GET_NUM_TEAMS)( void )
 {
@@ -754,29 +903,35 @@ xexpand(FTN_GET_TEAM_NUM)( void )
     #endif
 }
 
-#if KMP_MIC || KMP_OS_DARWIN
-
-static int __kmp_default_device = 0;
-
 int FTN_STDCALL
-FTN_GET_DEFAULT_DEVICE( void )
+xexpand(FTN_GET_DEFAULT_DEVICE)( void )
 {
-    return __kmp_default_device;
+    #if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
+        return 0;
+    #else
+        return __kmp_entry_thread() -> th.th_current_task -> td_icvs.default_device;
+    #endif
 }
 
 void FTN_STDCALL
-FTN_SET_DEFAULT_DEVICE( int KMP_DEREF arg )
+xexpand(FTN_SET_DEFAULT_DEVICE)( int KMP_DEREF arg )
 {
-    __kmp_default_device = KMP_DEREF arg;
+    #if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
+    // Nothing.
+    #else
+        __kmp_entry_thread() -> th.th_current_task -> td_icvs.default_device = KMP_DEREF arg;
+    #endif
 }
 
+#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
+
 int FTN_STDCALL
 FTN_GET_NUM_DEVICES( void )
 {
     return 0;
 }
 
-#endif // KMP_MIC || KMP_OS_DARWIN
+#endif // KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
 
 #if ! KMP_OS_LINUX
 
@@ -806,6 +961,63 @@ xexpand(FTN_IS_INITIAL_DEVICE)( void )
 
 #endif // OMP_40_ENABLED
 
+#if OMP_45_ENABLED && defined(KMP_STUB)
+// OpenMP 4.5 entries for stubs library
+
+int FTN_STDCALL
+FTN_GET_INITIAL_DEVICE(void)
+{
+   return -1;
+}
+
+// As all *target* functions are C-only parameters always passed by value
+void * FTN_STDCALL
+FTN_TARGET_ALLOC(size_t size, int device_num)
+{
+    return 0;
+}
+
+void FTN_STDCALL
+FTN_TARGET_FREE(void * device_ptr, int device_num)
+{
+}
+
+int FTN_STDCALL
+FTN_TARGET_IS_PRESENT(void * ptr, int device_num)
+{
+   return 0;
+}
+
+int FTN_STDCALL
+FTN_TARGET_MEMCPY(void *dst, void *src, size_t length, size_t  dst_offset,
+                  size_t src_offset, int dst_device, int src_device)
+{
+    return -1;
+}
+
+int FTN_STDCALL
+FTN_TARGET_MEMCPY_RECT(void *dst, void *src, size_t element_size, int num_dims,
+                       const size_t *volume, const size_t *dst_offsets,
+                       const size_t *src_offsets, const size_t *dst_dimensions,
+                       const size_t *src_dimensions, int dst_device, int src_device)
+{
+    return -1;
+}
+
+int FTN_STDCALL
+FTN_TARGET_ASSOCIATE_PTR(void *host_ptr, void *device_ptr, size_t  size,
+                         size_t  device_offset, int device_num)
+{
+    return -1;
+}
+
+int FTN_STDCALL
+FTN_TARGET_DISASSOCIATE_PTR(void *host_ptr, int device_num)
+{
+    return -1;
+}
+#endif // OMP_45_ENABLED && defined(KMP_STUB)
+
 #ifdef KMP_STUB
 typedef enum { UNINIT = -1, UNLOCKED, LOCKED } kmp_stub_lock_t;
 #endif /* KMP_STUB */
@@ -1008,6 +1220,13 @@ FTN_MALLOC( size_t KMP_DEREF size )
     return kmpc_malloc( KMP_DEREF size );
 }
 
+void * FTN_STDCALL
+FTN_ALIGNED_MALLOC( size_t KMP_DEREF size, size_t KMP_DEREF alignment )
+{
+    // kmpc_aligned_malloc initializes the library if needed
+    return kmpc_aligned_malloc( KMP_DEREF size, KMP_DEREF alignment );
+}
+
 void * FTN_STDCALL
 FTN_CALLOC( size_t KMP_DEREF nelem, size_t KMP_DEREF elsize )
 {
@@ -1089,6 +1308,22 @@ FTN_GET_CANCELLATION_STATUS(int cancel_kind) {
 
 #endif // OMP_40_ENABLED
 
+#if OMP_45_ENABLED
+/* returns the maximum allowed task priority */
+int FTN_STDCALL
+FTN_GET_MAX_TASK_PRIORITY( void )
+{
+#ifdef KMP_STUB
+    return 0;
+#else
+    if ( ! __kmp_init_serial ) {
+        __kmp_serial_initialize();
+    }
+    return __kmp_max_task_priority;
+#endif
+}
+#endif
+
 // GCC compatibility (versioned symbols)
 #ifdef KMP_USE_VERSION_SYMBOLS
 
@@ -1164,11 +1399,13 @@ xaliasify(FTN_GET_PROC_BIND, 40);
 xaliasify(FTN_GET_NUM_TEAMS, 40);
 xaliasify(FTN_GET_TEAM_NUM, 40);
 xaliasify(FTN_GET_CANCELLATION, 40);
+xaliasify(FTN_GET_DEFAULT_DEVICE, 40);
+xaliasify(FTN_SET_DEFAULT_DEVICE, 40);
 xaliasify(FTN_IS_INITIAL_DEVICE, 40);
 #endif /* OMP_40_ENABLED */
 
-#if OMP_41_ENABLED
-// OMP_4.1 aliases
+#if OMP_45_ENABLED
+// OMP_4.5 aliases
 #endif
 
 #if OMP_50_ENABLED
@@ -1229,15 +1466,17 @@ xversionify(FTN_IN_FINAL,          31, "OMP_3.1");
 
 #if OMP_40_ENABLED
 // OMP_4.0 versioned symbols
-xversionify(FTN_GET_PROC_BIND,     40, "OMP_4.0");
-xversionify(FTN_GET_NUM_TEAMS,     40, "OMP_4.0");
-xversionify(FTN_GET_TEAM_NUM,      40, "OMP_4.0");
-xversionify(FTN_GET_CANCELLATION,  40, "OMP_4.0");
-xversionify(FTN_IS_INITIAL_DEVICE, 40, "OMP_4.0");
+xversionify(FTN_GET_PROC_BIND,      40, "OMP_4.0");
+xversionify(FTN_GET_NUM_TEAMS,      40, "OMP_4.0");
+xversionify(FTN_GET_TEAM_NUM,       40, "OMP_4.0");
+xversionify(FTN_GET_CANCELLATION,   40, "OMP_4.0");
+xversionify(FTN_GET_DEFAULT_DEVICE, 40, "OMP_4.0");
+xversionify(FTN_SET_DEFAULT_DEVICE, 40, "OMP_4.0");
+xversionify(FTN_IS_INITIAL_DEVICE,  40, "OMP_4.0");
 #endif /* OMP_40_ENABLED */
 
-#if OMP_41_ENABLED
-// OMP_4.1 versioned symbols
+#if OMP_45_ENABLED
+// OMP_4.5 versioned symbols
 #endif
 
 #if OMP_50_ENABLED

runtime/src/kmp_ftn_extra.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_ftn_extra.c -- Fortran 'extra' linkage support for OpenMP.
+ * kmp_ftn_extra.cpp -- Fortran 'extra' linkage support for OpenMP.
  */
 
 
@@ -14,6 +14,7 @@
 
 
 #include "kmp.h"
+#include "kmp_affinity.h"
 
 #if KMP_OS_WINDOWS
 #   define KMP_FTN_ENTRIES KMP_FTN_PLAIN

runtime/src/kmp_ftn_os.h

@@ -35,6 +35,7 @@
     #define FTN_SET_LIBRARY                      kmp_set_library
     #define FTN_GET_LIBRARY                      kmp_get_library
     #define FTN_SET_DEFAULTS                     kmp_set_defaults
+    #define FTN_SET_DISP_NUM_BUFFERS             kmp_set_disp_num_buffers
     #define FTN_SET_AFFINITY                     kmp_set_affinity
     #define FTN_GET_AFFINITY                     kmp_get_affinity
     #define FTN_GET_AFFINITY_MAX_PROC            kmp_get_affinity_max_proc
@@ -45,6 +46,7 @@
     #define FTN_GET_AFFINITY_MASK_PROC           kmp_get_affinity_mask_proc
 
     #define FTN_MALLOC                           kmp_malloc
+    #define FTN_ALIGNED_MALLOC                   kmp_aligned_malloc
     #define FTN_CALLOC                           kmp_calloc
     #define FTN_REALLOC                          kmp_realloc
     #define FTN_FREE                             kmp_free
@@ -100,11 +102,11 @@
     #define FTN_GET_WTICK                        omp_get_wtick
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN
-    #define FTN_GET_DEFAULT_DEVICE               omp_get_default_device
-    #define FTN_SET_DEFAULT_DEVICE               omp_set_default_device
+#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
     #define FTN_GET_NUM_DEVICES                  omp_get_num_devices
 #endif
+    #define FTN_GET_DEFAULT_DEVICE               omp_get_default_device
+    #define FTN_SET_DEFAULT_DEVICE               omp_set_default_device
     #define FTN_IS_INITIAL_DEVICE                omp_is_initial_device
 #endif
 
@@ -113,6 +115,26 @@
     #define FTN_GET_CANCELLATION_STATUS          kmp_get_cancellation_status
 #endif
 
+#if OMP_45_ENABLED
+    #define FTN_GET_MAX_TASK_PRIORITY            omp_get_max_task_priority
+    #define FTN_GET_NUM_PLACES                   omp_get_num_places
+    #define FTN_GET_PLACE_NUM_PROCS              omp_get_place_num_procs
+    #define FTN_GET_PLACE_PROC_IDS               omp_get_place_proc_ids
+    #define FTN_GET_PLACE_NUM                    omp_get_place_num
+    #define FTN_GET_PARTITION_NUM_PLACES         omp_get_partition_num_places
+    #define FTN_GET_PARTITION_PLACE_NUMS         omp_get_partition_place_nums
+# ifdef KMP_STUB
+    #define FTN_GET_INITIAL_DEVICE               omp_get_initial_device
+    #define FTN_TARGET_ALLOC                     omp_target_alloc
+    #define FTN_TARGET_FREE                      omp_target_free
+    #define FTN_TARGET_IS_PRESENT                omp_target_is_present
+    #define FTN_TARGET_MEMCPY                    omp_target_memcpy
+    #define FTN_TARGET_MEMCPY_RECT               omp_target_memcpy_rect
+    #define FTN_TARGET_ASSOCIATE_PTR             omp_target_associate_ptr
+    #define FTN_TARGET_DISASSOCIATE_PTR          omp_target_disassociate_ptr
+# endif
+#endif
+
 #endif /* KMP_FTN_PLAIN */
 
 /* ------------------------------------------------------------------------ */
@@ -131,6 +153,7 @@
     #define FTN_SET_LIBRARY                      kmp_set_library_
     #define FTN_GET_LIBRARY                      kmp_get_library_
     #define FTN_SET_DEFAULTS                     kmp_set_defaults_
+    #define FTN_SET_DISP_NUM_BUFFERS             kmp_set_disp_num_buffers_
     #define FTN_SET_AFFINITY                     kmp_set_affinity_
     #define FTN_GET_AFFINITY                     kmp_get_affinity_
     #define FTN_GET_AFFINITY_MAX_PROC            kmp_get_affinity_max_proc_
@@ -141,6 +164,7 @@
     #define FTN_GET_AFFINITY_MASK_PROC           kmp_get_affinity_mask_proc_
 
     #define FTN_MALLOC                           kmp_malloc_
+    #define FTN_ALIGNED_MALLOC                   kmp_aligned_malloc_
     #define FTN_CALLOC                           kmp_calloc_
     #define FTN_REALLOC                          kmp_realloc_
     #define FTN_FREE                             kmp_free_
@@ -196,11 +220,11 @@
     #define FTN_GET_WTICK                        omp_get_wtick_
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN
-    #define FTN_GET_DEFAULT_DEVICE               omp_get_default_device_
-    #define FTN_SET_DEFAULT_DEVICE               omp_set_default_device_
+#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
     #define FTN_GET_NUM_DEVICES                  omp_get_num_devices_
 #endif
+    #define FTN_GET_DEFAULT_DEVICE               omp_get_default_device_
+    #define FTN_SET_DEFAULT_DEVICE               omp_set_default_device_
     #define FTN_IS_INITIAL_DEVICE                omp_is_initial_device_
 #endif
 
@@ -210,6 +234,26 @@
     #define FTN_GET_CANCELLATION_STATUS          kmp_get_cancellation_status_
 #endif
 
+#if OMP_45_ENABLED
+    #define FTN_GET_MAX_TASK_PRIORITY            omp_get_max_task_priority_
+    #define FTN_GET_NUM_PLACES                   omp_get_num_places_
+    #define FTN_GET_PLACE_NUM_PROCS              omp_get_place_num_procs_
+    #define FTN_GET_PLACE_PROC_IDS               omp_get_place_proc_ids_
+    #define FTN_GET_PLACE_NUM                    omp_get_place_num_
+    #define FTN_GET_PARTITION_NUM_PLACES         omp_get_partition_num_places_
+    #define FTN_GET_PARTITION_PLACE_NUMS         omp_get_partition_place_nums_
+# ifdef KMP_STUB
+    #define FTN_GET_INITIAL_DEVICE               omp_get_initial_device_
+    #define FTN_TARGET_ALLOC                     omp_target_alloc_
+    #define FTN_TARGET_FREE                      omp_target_free_
+    #define FTN_TARGET_IS_PRESENT                omp_target_is_present_
+    #define FTN_TARGET_MEMCPY                    omp_target_memcpy_
+    #define FTN_TARGET_MEMCPY_RECT               omp_target_memcpy_rect_
+    #define FTN_TARGET_ASSOCIATE_PTR             omp_target_associate_ptr_
+    #define FTN_TARGET_DISASSOCIATE_PTR          omp_target_disassociate_ptr_
+# endif
+#endif
+
 #endif /* KMP_FTN_APPEND */
 
 /* ------------------------------------------------------------------------ */
@@ -228,6 +272,7 @@
     #define FTN_SET_LIBRARY                      KMP_SET_LIBRARY
     #define FTN_GET_LIBRARY                      KMP_GET_LIBRARY
     #define FTN_SET_DEFAULTS                     KMP_SET_DEFAULTS
+    #define FTN_SET_DISP_NUM_BUFFERS             KMP_SET_DISP_NUM_BUFFERS
     #define FTN_SET_AFFINITY                     KMP_SET_AFFINITY
     #define FTN_GET_AFFINITY                     KMP_GET_AFFINITY
     #define FTN_GET_AFFINITY_MAX_PROC            KMP_GET_AFFINITY_MAX_PROC
@@ -238,6 +283,7 @@
     #define FTN_GET_AFFINITY_MASK_PROC           KMP_GET_AFFINITY_MASK_PROC
 
     #define FTN_MALLOC                           KMP_MALLOC
+    #define FTN_ALIGNED_MALLOC                   KMP_ALIGNED_MALLOC
     #define FTN_CALLOC                           KMP_CALLOC
     #define FTN_REALLOC                          KMP_REALLOC
     #define FTN_FREE                             KMP_FREE
@@ -293,11 +339,11 @@
     #define FTN_GET_WTICK                        OMP_GET_WTICK
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN
-    #define FTN_GET_DEFAULT_DEVICE               OMP_GET_DEFAULT_DEVICE
-    #define FTN_SET_DEFAULT_DEVICE               OMP_SET_DEFAULT_DEVICE
+#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
     #define FTN_GET_NUM_DEVICES                  OMP_GET_NUM_DEVICES
 #endif
+    #define FTN_GET_DEFAULT_DEVICE               OMP_GET_DEFAULT_DEVICE
+    #define FTN_SET_DEFAULT_DEVICE               OMP_SET_DEFAULT_DEVICE
     #define FTN_IS_INITIAL_DEVICE                OMP_IS_INITIAL_DEVICE
 #endif
 
@@ -307,6 +353,26 @@
     #define FTN_GET_CANCELLATION_STATUS          KMP_GET_CANCELLATION_STATUS
 #endif
 
+#if OMP_45_ENABLED
+    #define FTN_GET_MAX_TASK_PRIORITY            OMP_GET_MAX_TASK_PRIORITY
+    #define FTN_GET_NUM_PLACES                   OMP_GET_NUM_PLACES
+    #define FTN_GET_PLACE_NUM_PROCS              OMP_GET_PLACE_NUM_PROCS
+    #define FTN_GET_PLACE_PROC_IDS               OMP_GET_PLACE_PROC_IDS
+    #define FTN_GET_PLACE_NUM                    OMP_GET_PLACE_NUM
+    #define FTN_GET_PARTITION_NUM_PLACES         OMP_GET_PARTITION_NUM_PLACES
+    #define FTN_GET_PARTITION_PLACE_NUMS         OMP_GET_PARTITION_PLACE_NUMS
+# ifdef KMP_STUB
+    #define FTN_GET_INITIAL_DEVICE               OMP_GET_INITIAL_DEVICE
+    #define FTN_TARGET_ALLOC                     OMP_TARGET_ALLOC
+    #define FTN_TARGET_FREE                      OMP_TARGET_FREE
+    #define FTN_TARGET_IS_PRESENT                OMP_TARGET_IS_PRESENT
+    #define FTN_TARGET_MEMCPY                    OMP_TARGET_MEMCPY
+    #define FTN_TARGET_MEMCPY_RECT               OMP_TARGET_MEMCPY_RECT
+    #define FTN_TARGET_ASSOCIATE_PTR             OMP_TARGET_ASSOCIATE_PTR
+    #define FTN_TARGET_DISASSOCIATE_PTR          OMP_TARGET_DISASSOCIATE_PTR
+# endif
+#endif
+
 #endif /* KMP_FTN_UPPER */
 
 /* ------------------------------------------------------------------------ */
@@ -325,6 +391,7 @@
     #define FTN_SET_LIBRARY                      KMP_SET_LIBRARY_
     #define FTN_GET_LIBRARY                      KMP_GET_LIBRARY_
     #define FTN_SET_DEFAULTS                     KMP_SET_DEFAULTS_
+    #define FTN_SET_DISP_NUM_BUFFERS             KMP_SET_DISP_NUM_BUFFERS_
     #define FTN_SET_AFFINITY                     KMP_SET_AFFINITY_
     #define FTN_GET_AFFINITY                     KMP_GET_AFFINITY_
     #define FTN_GET_AFFINITY_MAX_PROC            KMP_GET_AFFINITY_MAX_PROC_
@@ -335,6 +402,7 @@
     #define FTN_GET_AFFINITY_MASK_PROC           KMP_GET_AFFINITY_MASK_PROC_
 
     #define FTN_MALLOC                           KMP_MALLOC_
+    #define FTN_ALIGNED_MALLOC                   KMP_ALIGNED_MALLOC_
     #define FTN_CALLOC                           KMP_CALLOC_
     #define FTN_REALLOC                          KMP_REALLOC_
     #define FTN_FREE                             KMP_FREE_
@@ -390,11 +458,11 @@
     #define FTN_GET_WTICK                        OMP_GET_WTICK_
 
 #if OMP_40_ENABLED
-#if KMP_MIC || KMP_OS_DARWIN
-    #define FTN_GET_DEFAULT_DEVICE               OMP_GET_DEFAULT_DEVICE_
-    #define FTN_SET_DEFAULT_DEVICE               OMP_SET_DEFAULT_DEVICE_
+#if KMP_MIC || KMP_OS_DARWIN || defined(KMP_STUB)
     #define FTN_GET_NUM_DEVICES                  OMP_GET_NUM_DEVICES_
 #endif
+    #define FTN_GET_DEFAULT_DEVICE               OMP_GET_DEFAULT_DEVICE_
+    #define FTN_SET_DEFAULT_DEVICE               OMP_SET_DEFAULT_DEVICE_
     #define FTN_IS_INITIAL_DEVICE                OMP_IS_INITIAL_DEVICE_
 #endif
 
@@ -404,6 +472,26 @@
     #define FTN_GET_CANCELLATION_STATUS          KMP_GET_CANCELLATION_STATUS_
 #endif
 
+#if OMP_45_ENABLED
+    #define FTN_GET_MAX_TASK_PRIORITY            OMP_GET_MAX_TASK_PRIORITY_
+    #define FTN_GET_NUM_PLACES                   OMP_GET_NUM_PLACES_
+    #define FTN_GET_PLACE_NUM_PROCS              OMP_GET_PLACE_NUM_PROCS_
+    #define FTN_GET_PLACE_PROC_IDS               OMP_GET_PLACE_PROC_IDS_
+    #define FTN_GET_PLACE_NUM                    OMP_GET_PLACE_NUM_
+    #define FTN_GET_PARTITION_NUM_PLACES         OMP_GET_PARTITION_NUM_PLACES_
+    #define FTN_GET_PARTITION_PLACE_NUMS         OMP_GET_PARTITION_PLACE_NUMS_
+# ifdef KMP_STUB
+    #define FTN_GET_INITIAL_DEVICE               OMP_GET_INITIAL_DEVICE_
+    #define FTN_TARGET_ALLOC                     OMP_TARGET_ALLOC_
+    #define FTN_TARGET_FREE                      OMP_TARGET_FREE_
+    #define FTN_TARGET_IS_PRESENT                OMP_TARGET_IS_PRESENT_
+    #define FTN_TARGET_MEMCPY                    OMP_TARGET_MEMCPY_
+    #define FTN_TARGET_MEMCPY_RECT               OMP_TARGET_MEMCPY_RECT_
+    #define FTN_TARGET_ASSOCIATE_PTR             OMP_TARGET_ASSOCIATE_PTR_
+    #define FTN_TARGET_DISASSOCIATE_PTR          OMP_TARGET_DISASSOCIATE_PTR_
+# endif
+#endif
+
 #endif /* KMP_FTN_UAPPEND */
 
 /* ------------------------------------------------------------------ */
@@ -474,7 +562,7 @@
 // All GOMP_3.0 symbols
 #define KMP_API_NAME_GOMP_TASKYIELD                      GOMP_taskyield
 
-// All GOMP_4.0 symbols 
+// All GOMP_4.0 symbols
 // TODO: As of 2013-10-14, none of the GOMP_4.0 functions are implemented in libomp
 #define KMP_API_NAME_GOMP_BARRIER_CANCEL                 GOMP_barrier_cancel
 #define KMP_API_NAME_GOMP_CANCEL                         GOMP_cancel

runtime/src/kmp_ftn_stdcall.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_ftn_stdcall.c -- Fortran __stdcall linkage support for OpenMP.
+ * kmp_ftn_stdcall.cpp -- Fortran __stdcall linkage support for OpenMP.
  */
 
 

runtime/src/kmp_global.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_global.c -- KPTS global variables for runtime support library
+ * kmp_global.cpp -- KPTS global variables for runtime support library
  */
 
 
@@ -14,10 +14,13 @@
 
 
 #include "kmp.h"
+#include "kmp_affinity.h"
 
 kmp_key_t __kmp_gtid_threadprivate_key;
 
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
 kmp_cpuinfo_t   __kmp_cpuinfo = { 0 }; // Not initialized
+#endif
 
 #if KMP_STATS_ENABLED
 #include "kmp_stats.h"
@@ -25,18 +28,14 @@ kmp_cpuinfo_t   __kmp_cpuinfo = { 0 }; // Not initialized
 kmp_tas_lock_t __kmp_stats_lock;
 
 // global list of per thread stats, the head is a sentinel node which accumulates all stats produced before __kmp_create_worker is called.
-kmp_stats_list __kmp_stats_list;
+kmp_stats_list* __kmp_stats_list;
 
 // thread local pointer to stats node within list
-__thread kmp_stats_list* __kmp_stats_thread_ptr = &__kmp_stats_list;
+__thread kmp_stats_list* __kmp_stats_thread_ptr = NULL;
 
 // gives reference tick for all events (considered the 0 tick)
 tsc_tick_count __kmp_stats_start_time;
 #endif
-#if KMP_USE_HWLOC
-int __kmp_hwloc_error = FALSE;
-hwloc_topology_t __kmp_hwloc_topology = NULL;
-#endif
 
 /* ----------------------------------------------------- */
 /* INITIALIZATION VARIABLES */
@@ -46,7 +45,9 @@ volatile int        __kmp_init_gtid       = FALSE;
 volatile int        __kmp_init_common     = FALSE;
 volatile int        __kmp_init_middle     = FALSE;
 volatile int        __kmp_init_parallel   = FALSE;
+#if KMP_USE_MONITOR
 volatile int        __kmp_init_monitor    = 0;  /* 1 - launched, 2 - actually started (Windows* OS only) */
+#endif
 volatile int        __kmp_init_user_locks = FALSE;
 
 /* list of address of allocated caches for commons */
@@ -63,7 +64,9 @@ unsigned int __kmp_init_wait = KMP_DEFAULT_INIT_WAIT;   /* initial number of spi
 unsigned int __kmp_next_wait = KMP_DEFAULT_NEXT_WAIT;   /* susequent number of spin-tests */
 
 size_t      __kmp_stksize         = KMP_DEFAULT_STKSIZE;
+#if KMP_USE_MONITOR
 size_t      __kmp_monitor_stksize = 0;  // auto adjust
+#endif
 size_t      __kmp_stkoffset       = KMP_DEFAULT_STKOFFSET;
 int         __kmp_stkpadding      = KMP_MIN_STKPADDING;
 
@@ -126,6 +129,7 @@ int      __kmp_dflt_team_nth_ub = 0;
 int           __kmp_tp_capacity = 0;
 int             __kmp_tp_cached = 0;
 int           __kmp_dflt_nested = FALSE;
+int  __kmp_dispatch_num_buffers = KMP_DFLT_DISP_NUM_BUFF;
 int __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; /* max_active_levels limit */
 #if KMP_NESTED_HOT_TEAMS
 int __kmp_hot_teams_mode         = 0; /* 0 - free extra threads when reduced */
@@ -138,8 +142,10 @@ enum sched_type    __kmp_static = kmp_sch_static_greedy; /* default static sched
 enum sched_type    __kmp_guided = kmp_sch_guided_iterative_chunked; /* default guided scheduling method */
 enum sched_type      __kmp_auto = kmp_sch_guided_analytical_chunked; /* default auto scheduling method */
 int        __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
+#if KMP_USE_MONITOR
 int       __kmp_monitor_wakeups = KMP_MIN_MONITOR_WAKEUPS;
 int          __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( KMP_DEFAULT_BLOCKTIME, KMP_MIN_MONITOR_WAKEUPS );
+#endif
 #ifdef KMP_ADJUST_BLOCKTIME
 int               __kmp_zero_bt = FALSE;
 #endif /* KMP_ADJUST_BLOCKTIME */
@@ -217,16 +223,22 @@ enum mic_type __kmp_mic_type = non_mic;
 
 #if KMP_AFFINITY_SUPPORTED
 
-# if KMP_GROUP_AFFINITY
+KMPAffinity* __kmp_affinity_dispatch = NULL;
 
+# if KMP_USE_HWLOC
+int __kmp_hwloc_error = FALSE;
+hwloc_topology_t __kmp_hwloc_topology = NULL;
+# endif
+
+# if KMP_OS_WINDOWS
+#  if KMP_GROUP_AFFINITY
 int __kmp_num_proc_groups = 1;
-
+#  endif /* KMP_GROUP_AFFINITY */
 kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount = NULL;
 kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount = NULL;
 kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity = NULL;
 kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity = NULL;
-
-# endif /* KMP_GROUP_AFFINITY */
+# endif /* KMP_OS_WINDOWS */
 
 size_t   __kmp_affin_mask_size = 0;
 enum affinity_type __kmp_affinity_type = affinity_default;
@@ -258,7 +270,14 @@ int __kmp_place_num_cores = 0;
 int __kmp_place_core_offset = 0;
 int __kmp_place_num_threads_per_core = 0;
 
+#if OMP_40_ENABLED
+kmp_int32 __kmp_default_device = 0;
+#endif
+
 kmp_tasking_mode_t __kmp_tasking_mode = tskm_task_teams;
+#if OMP_45_ENABLED
+kmp_int32 __kmp_max_task_priority = 0;
+#endif
 
 /* This check ensures that the compiler is passing the correct data type
  * for the flags formal parameter of the function kmpc_omp_task_alloc().
@@ -332,6 +351,8 @@ int        __kmp_env_consistency_check  = FALSE;  /* KMP_CONSISTENCY_CHECK speci
 
 kmp_uint32 __kmp_yield_init = KMP_INIT_WAIT;
 kmp_uint32 __kmp_yield_next = KMP_NEXT_WAIT;
+
+#if KMP_USE_MONITOR
 kmp_uint32 __kmp_yielding_on = 1;
 #if KMP_OS_CNK
 kmp_uint32 __kmp_yield_cycle = 0;
@@ -340,6 +361,7 @@ kmp_uint32 __kmp_yield_cycle = 1;     /* Yield-cycle is on by default */
 #endif
 kmp_int32  __kmp_yield_on_count = 10; /* By default, yielding is on for 10 monitor periods. */
 kmp_int32  __kmp_yield_off_count = 1; /* By default, yielding is off for 1 monitor periods. */
+#endif
 /* ----------------------------------------------------- */
 
 
@@ -387,8 +409,10 @@ KMP_ALIGN_CACHE_INTERNODE
 kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
 KMP_ALIGN_CACHE_INTERNODE
 kmp_bootstrap_lock_t __kmp_exit_lock;   /* exit() is not always thread-safe */
+#if KMP_USE_MONITOR
 KMP_ALIGN_CACHE_INTERNODE
 kmp_bootstrap_lock_t __kmp_monitor_lock; /* control monitor thread creation */
+#endif
 KMP_ALIGN_CACHE_INTERNODE
 kmp_bootstrap_lock_t __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and __kmp_threads expansion to co-exist */
 
@@ -404,7 +428,9 @@ KMP_ALIGN_CACHE
 kmp_bootstrap_lock_t __kmp_initz_lock   = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_initz_lock   ); /* Control initializations */
 kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
 kmp_bootstrap_lock_t __kmp_exit_lock;   /* exit() is not always thread-safe */
+#if KMP_USE_MONITOR
 kmp_bootstrap_lock_t __kmp_monitor_lock; /* control monitor thread creation */
+#endif
 kmp_bootstrap_lock_t __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and __kmp_threads expansion to co-exist */
 
 KMP_ALIGN(128)

runtime/src/kmp_gsupport.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_gsupport.c
+ * kmp_gsupport.cpp
  */
 
 
@@ -35,6 +35,13 @@ xexpand(KMP_API_NAME_GOMP_BARRIER)(void)
     int gtid = __kmp_entry_gtid();
     MKLOC(loc, "GOMP_barrier");
     KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid));
+#if OMPT_SUPPORT && OMPT_TRACE
+    ompt_frame_t * ompt_frame;
+    if (ompt_enabled ) {
+        ompt_frame = __ompt_get_task_frame_internal(0);
+        ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
+    } 
+#endif
     __kmpc_barrier(&loc, gtid);
 }
 
@@ -225,7 +232,7 @@ xexpand(KMP_API_NAME_GOMP_ORDERED_END)(void)
 // (IA-32 architecture) or 64-bit signed (Intel(R) 64).
 //
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS
 # define KMP_DISPATCH_INIT              __kmp_aux_dispatch_init_4
 # define KMP_DISPATCH_FINI_CHUNK        __kmp_aux_dispatch_fini_chunk_4
 # define KMP_DISPATCH_NEXT              __kmpc_dispatch_next_4
@@ -364,7 +371,7 @@ __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), mic
         __kmp_run_before_invoked_task(gtid, tid, thr, team);
     }
 
-#if OMPT_SUPPORT 
+#if OMPT_SUPPORT
     if (ompt_enabled) {
 #if OMPT_TRACE
         ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
@@ -384,34 +391,37 @@ __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), mic
 static void
 __kmp_GOMP_serialized_parallel(ident_t *loc, kmp_int32 gtid, void (*task)(void *))
 {
-    __kmp_serialized_parallel(loc, gtid);
-
 #if OMPT_SUPPORT
+    ompt_parallel_id_t ompt_parallel_id;
     if (ompt_enabled) {
-        ompt_task_id_t ompt_task_id = __ompt_get_task_id_internal(0);
-        ompt_frame_t  *ompt_frame = __ompt_get_task_frame_internal(0);
-        kmp_info_t *thr = __kmp_threads[gtid];
+        ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
 
-        ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(gtid);
-        ompt_task_id_t my_ompt_task_id = __ompt_task_id_new(gtid);
-
-        ompt_frame->exit_runtime_frame = NULL;
+        ompt_parallel_id = __ompt_parallel_id_new(gtid);
 
         // parallel region callback
         if (ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) {
             int team_size = 1;
             ompt_callbacks.ompt_callback(ompt_event_parallel_begin)(
-                ompt_task_id, ompt_frame, ompt_parallel_id,
+                task_info->task_id, &task_info->frame, ompt_parallel_id,
                 team_size, (void *) task,
                 OMPT_INVOKER(fork_context_gnu));
         }
+    }
+#endif
+
+    __kmp_serialized_parallel(loc, gtid);
+
+#if OMPT_SUPPORT
+    if (ompt_enabled) {
+        kmp_info_t *thr = __kmp_threads[gtid];
+
+        ompt_task_id_t my_ompt_task_id = __ompt_task_id_new(gtid);
 
         // set up lightweight task
         ompt_lw_taskteam_t *lwt = (ompt_lw_taskteam_t *)
             __kmp_allocate(sizeof(ompt_lw_taskteam_t));
         __ompt_lw_taskteam_init(lwt, thr, gtid, (void *) task, ompt_parallel_id);
         lwt->ompt_task_info.task_id = my_ompt_task_id;
-        lwt->ompt_task_info.frame.exit_runtime_frame = 0;
         __ompt_lw_taskteam_link(lwt, thr);
 
 #if OMPT_TRACE
@@ -433,11 +443,11 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), void *data, unsi
     int gtid = __kmp_entry_gtid();
 
 #if OMPT_SUPPORT
-    ompt_frame_t *parent_frame;
+    ompt_frame_t *parent_frame, *frame;
 
     if (ompt_enabled) {
         parent_frame = __ompt_get_task_frame_internal(0);
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0);
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
@@ -457,7 +467,8 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), void *data, unsi
 
 #if OMPT_SUPPORT
     if (ompt_enabled) {
-        parent_frame->reenter_runtime_frame = NULL;
+        frame = __ompt_get_task_frame_internal(0);
+        frame->exit_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 }
@@ -477,41 +488,22 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_END)(void)
 
 #if OMPT_SUPPORT
     ompt_parallel_id_t parallel_id;
+    ompt_task_id_t serialized_task_id;
     ompt_frame_t *ompt_frame = NULL;
 
     if (ompt_enabled) {
         ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
         parallel_id = team_info->parallel_id;
 
-        // Record that we re-entered the runtime system in the implicit
-        // task frame representing the parallel region. 
-        ompt_frame = __ompt_get_task_frame_internal(0);
-        ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
-
-#if OMPT_TRACE
-        if (ompt_enabled &&
-            ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
-            ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
-            ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
-                parallel_id, task_info->task_id);
-        }
-#endif
+        ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
+        serialized_task_id = task_info->task_id;
 
         // unlink if necessary. no-op if there is not a lightweight task.
         ompt_lw_taskteam_t *lwt = __ompt_lw_taskteam_unlink(thr);
         // GOMP allocates/frees lwt since it can't be kept on the stack
-        if (lwt) { 
+        if (lwt) {
            __kmp_free(lwt);
-     
-#if OMPT_SUPPORT
-           if (ompt_enabled) {
-              // Since a lightweight task was destroyed, make sure that the
-              // remaining deepest task knows the stack frame where the runtime 
-              // was reentered.
-              ompt_frame = __ompt_get_task_frame_internal(0);
-              ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
-           }
-#endif
+
         }
     }
 #endif
@@ -522,10 +514,10 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_END)(void)
 
 #if OMPT_SUPPORT
         if (ompt_enabled) {
-          // Set reenter frame in parent task, which will become current task
-          // in the midst of join. This is needed before the end_parallel callback.
-          ompt_frame = __ompt_get_task_frame_internal(1);
-          ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+          // Implicit task is finished here, in the barrier we might schedule deferred tasks, 
+          // these don't see the implicit task on the stack
+          ompt_frame = __ompt_get_task_frame_internal(0);
+          ompt_frame->exit_runtime_frame = NULL;
         }
 #endif
 
@@ -534,29 +526,31 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_END)(void)
             , fork_context_gnu
 #endif
         );
-#if OMPT_SUPPORT
-        if (ompt_enabled) {
-          ompt_frame->reenter_runtime_frame = NULL;
-        }
-#endif
     }
     else {
+#if OMPT_SUPPORT && OMPT_TRACE
+        if (ompt_enabled &&
+            ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
+            ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
+                parallel_id, serialized_task_id);
+        }
+#endif
+
         __kmpc_end_serialized_parallel(&loc, gtid);
 
 #if OMPT_SUPPORT
         if (ompt_enabled) {
-            // Record that we re-entered the runtime system in the frame that 
+            // Record that we re-entered the runtime system in the frame that
             // created the parallel region.
-            ompt_frame->reenter_runtime_frame = __builtin_frame_address(0);
+            ompt_task_info_t *parent_task_info = __ompt_get_taskinfo(0);
 
             if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
-                ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
                 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
-                    parallel_id, task_info->task_id, 
+                    parallel_id, parent_task_info->task_id,
                     OMPT_INVOKER(fork_context_gnu));
             }
 
-            ompt_frame->reenter_runtime_frame = NULL;
+            parent_task_info->frame.reenter_runtime_frame = NULL;
 
             thr->th.ompt_thread_info.state =
                 (((thr->th.th_team)->t.t_serialized) ?
@@ -890,7 +884,7 @@ LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), \
     ompt_frame_t *parent_frame; \
     if (ompt_enabled) { \
         parent_frame = __ompt_get_task_frame_internal(0); \
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0); \
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1); \
     }
 
 
@@ -901,20 +895,20 @@ LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), \
 
 #else
 
-#define OMPT_LOOP_PRE() 
+#define OMPT_LOOP_PRE()
 
 #define OMPT_LOOP_POST()
 
 #endif
 
 
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START), 
+PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START),
                     kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START), 
+PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START),
                     kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START), 
+PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START),
                     kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START), 
+PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START),
                     kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
 
 
@@ -924,7 +918,7 @@ PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START),
 
 void
 xexpand(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, void (*copy_func)(void *, void *),
-  long arg_size, long arg_align, int if_cond, unsigned gomp_flags)
+  long arg_size, long arg_align, bool if_cond, unsigned gomp_flags)
 {
     MKLOC(loc, "GOMP_task");
     int gtid = __kmp_entry_gtid();
@@ -994,7 +988,7 @@ xexpand(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, void (*copy_fu
 #if OMPT_SUPPORT
         if (ompt_enabled) {
             thread->th.ompt_thread_info = oldInfo;
-            taskdata->ompt_task_info.frame.exit_runtime_frame = 0;
+            taskdata->ompt_task_info.frame.exit_runtime_frame = NULL;
         }
 #endif
     }
@@ -1093,7 +1087,7 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)(void (*task) (void *), void *
 
     if (ompt_enabled) {
         parent_frame = __ompt_get_task_frame_internal(0);
-        parent_frame->reenter_runtime_frame = __builtin_frame_address(0);
+        parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
     }
 #endif
 
@@ -1160,6 +1154,13 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), void *data, unsigned n
     MKLOC(loc, "GOMP_parallel");
     KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid));
 
+#if OMPT_SUPPORT
+    ompt_task_info_t *parent_task_info, *task_info;
+    if (ompt_enabled) {
+        parent_task_info = __ompt_get_taskinfo(0);
+        parent_task_info->frame.reenter_runtime_frame = __builtin_frame_address(1);
+    }
+#endif
     if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
         if (num_threads != 0) {
             __kmp_push_num_threads(&loc, gtid, num_threads);
@@ -1173,8 +1174,20 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), void *data, unsigned n
     else {
         __kmp_GOMP_serialized_parallel(&loc, gtid, task);
     }
+#if OMPT_SUPPORT
+    if (ompt_enabled) {
+        task_info = __ompt_get_taskinfo(0);
+        task_info->frame.exit_runtime_frame = __builtin_frame_address(0);
+    }
+#endif
     task(data);
     xexpand(KMP_API_NAME_GOMP_PARALLEL_END)();
+#if OMPT_SUPPORT
+    if (ompt_enabled) {
+        task_info->frame.exit_runtime_frame = NULL;
+        parent_task_info->frame.reenter_runtime_frame = NULL;
+    }
+#endif
 }
 
 void
@@ -1208,7 +1221,7 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task) (void *), void *data,
     KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid));
 }
 
-#define PARALLEL_LOOP(func, schedule) \
+#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post) \
     void func (void (*task) (void *), void *data, unsigned num_threads,      \
       long lb, long ub, long str, long chunk_sz, unsigned flags)             \
     {                                                                        \
@@ -1217,6 +1230,7 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task) (void *), void *data,
         KA_TRACE(20, ( #func ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n",        \
           gtid, lb, ub, str, chunk_sz ));                                    \
                                                                              \
+        ompt_pre();                                                          \
         if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {                 \
             if (num_threads != 0) {                                          \
                 __kmp_push_num_threads(&loc, gtid, num_threads);             \
@@ -1238,20 +1252,25 @@ xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task) (void *), void *data,
           (schedule) != kmp_sch_static);                                     \
         task(data);                                                          \
         xexpand(KMP_API_NAME_GOMP_PARALLEL_END)();                           \
+        ompt_post();                                                          \
                                                                              \
         KA_TRACE(20, ( #func " exit: T#%d\n", gtid));                        \
     }
 
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), kmp_sch_static)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), kmp_sch_dynamic_chunked)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), kmp_sch_guided_chunked)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), kmp_sch_runtime)
+PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), kmp_sch_static, 
+                OMPT_LOOP_PRE, OMPT_LOOP_POST)
+PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), kmp_sch_dynamic_chunked, 
+                OMPT_LOOP_PRE, OMPT_LOOP_POST)
+PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), kmp_sch_guided_chunked, 
+                OMPT_LOOP_PRE, OMPT_LOOP_POST)
+PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), kmp_sch_runtime, 
+                OMPT_LOOP_PRE, OMPT_LOOP_POST)
 
 
 void
 xexpand(KMP_API_NAME_GOMP_TASKGROUP_START)(void)
 {
-    int gtid = __kmp_get_gtid();
+    int gtid = __kmp_entry_gtid();
     MKLOC(loc, "GOMP_taskgroup_start");
     KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid));
 

runtime/src/kmp_i18n.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_i18n.c
+ * kmp_i18n.cpp
  */
 
 
@@ -109,13 +109,13 @@ __kmp_i18n_do_catopen(
     KMP_DEBUG_ASSERT( cat    == KMP_I18N_NULLCAT );
 
     english =
-	lang == NULL                       ||  // In all these cases English language is used.
-	strcmp( lang, "" )            == 0 ||
+        lang == NULL                       ||  // In all these cases English language is used.
+        strcmp( lang, "" )            == 0 ||
         strcmp( lang, " " )           == 0 ||
-              // Workaround for Fortran RTL bug DPD200137873 "Fortran runtime resets LANG env var
-              // to space if it is not set".
-	strcmp( lang, "C" )           == 0 ||
-	strcmp( lang, "POSIX" )       == 0;
+        // Workaround for Fortran RTL bug DPD200137873 "Fortran runtime resets LANG env var
+        // to space if it is not set".
+        strcmp( lang, "C" )           == 0 ||
+        strcmp( lang, "POSIX" )       == 0;
 
     if ( ! english ) {  // English language is not yet detected, let us continue.
         // Format of LANG is: [language[_territory][.codeset][@modifier]]
@@ -132,8 +132,8 @@ __kmp_i18n_do_catopen(
     // Do not try to open English catalog because internal messages are
     // exact copy of messages in English catalog.
     if ( english ) {
-	status = KMP_I18N_ABSENT;  // mark catalog as absent so it will not be re-opened.
-	return;
+        status = KMP_I18N_ABSENT;  // mark catalog as absent so it will not be re-opened.
+        return;
     }
 
     cat = catopen( name, 0 );
@@ -141,36 +141,41 @@ __kmp_i18n_do_catopen(
     status = ( cat == KMP_I18N_NULLCAT ? KMP_I18N_ABSENT : KMP_I18N_OPENED );
 
     if ( status == KMP_I18N_ABSENT ) {
-      if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
-        int    error   = errno; // Save errno immediately.
-	char * nlspath = __kmp_env_get( "NLSPATH" );
-        char * lang    = __kmp_env_get( "LANG" );
+        if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
+            int    error   = errno; // Save errno immediately.
+            char * nlspath = __kmp_env_get( "NLSPATH" );
+            char * lang    = __kmp_env_get( "LANG" );
 
-	// Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
-	// __kmp_i18n_catgets() will not try to open catalog, but will return default message.
-	__kmp_msg(
-	    kmp_ms_warning,
-	    KMP_MSG( CantOpenMessageCatalog, name ),
-	    KMP_ERR( error ),
-	    KMP_HNT( CheckEnvVar, "NLSPATH", nlspath ),
-            KMP_HNT( CheckEnvVar, "LANG", lang ),
-	    __kmp_msg_null
-	);
-	KMP_INFORM( WillUseDefaultMessages );
-        KMP_INTERNAL_FREE( nlspath );
-        KMP_INTERNAL_FREE( lang );
-      }
+            // Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
+            // __kmp_i18n_catgets() will not try to open catalog, but will return default message.
+            kmp_msg_t err_code = KMP_ERR( error );
+            __kmp_msg(
+                kmp_ms_warning,
+                KMP_MSG( CantOpenMessageCatalog, name ),
+                err_code,
+                KMP_HNT( CheckEnvVar, "NLSPATH", nlspath ),
+                KMP_HNT( CheckEnvVar, "LANG", lang ),
+                __kmp_msg_null
+            );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+                __kmp_str_free(&err_code.str);
+            }
+
+            KMP_INFORM( WillUseDefaultMessages );
+            KMP_INTERNAL_FREE( nlspath );
+            KMP_INTERNAL_FREE( lang );
+        }
     } else { // status == KMP_I18N_OPENED
 
         int section = get_section( kmp_i18n_prp_Version );
         int number  = get_number( kmp_i18n_prp_Version );
         char const * expected = __kmp_i18n_default_table.sect[ section ].str[ number ];
-            // Expected version of the catalog.
+        // Expected version of the catalog.
         kmp_str_buf_t version;   // Actual version of the catalog.
         __kmp_str_buf_init( & version );
         __kmp_str_buf_print( & version, "%s", catgets( cat, section, number, NULL ) );
 
-            // String returned by catgets is invalid after closing the catalog, so copy it.
+        // String returned by catgets is invalid after closing the catalog, so copy it.
         if ( strcmp( version.str, expected ) != 0 ) {
             __kmp_i18n_catclose();     // Close bad catalog.
             status = KMP_I18N_ABSENT;  // And mark it as absent.
@@ -317,7 +322,7 @@ kmp_i18n_table_free(
     table->size = 0;
     KMP_INTERNAL_FREE( (void *) table->sect );
     table->sect = NULL;
-} // kmp_i8n_table_free
+} // kmp_i18n_table_free
 
 
 void
@@ -325,7 +330,7 @@ __kmp_i18n_do_catopen(
 ) {
 
     LCID          locale_id = GetThreadLocale();
-    WORD 	  lang_id = LANGIDFROMLCID( locale_id );
+    WORD          lang_id = LANGIDFROMLCID( locale_id );
     WORD          primary_lang_id = PRIMARYLANGID( lang_id );
     kmp_str_buf_t path;
 
@@ -337,8 +342,8 @@ __kmp_i18n_do_catopen(
     // Do not try to open English catalog because internal messages are
     // exact copy of messages in English catalog.
     if ( primary_lang_id == LANG_ENGLISH ) {
-	status = KMP_I18N_ABSENT;  // mark catalog as absent so it will not be re-opened.
-	goto end;
+        status = KMP_I18N_ABSENT;  // mark catalog as absent so it will not be re-opened.
+        goto end;
     }; // if
 
     // Construct resource DLL name.
@@ -395,30 +400,35 @@ __kmp_i18n_do_catopen(
 
     if ( status == KMP_I18N_ABSENT ) {
       if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
-	DWORD error = GetLastError();
-	// Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
-	// __kmp_i18n_catgets() will not try to open catalog but will return default message.
+        DWORD error = GetLastError();
+        // Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
+        // __kmp_i18n_catgets() will not try to open catalog but will return default message.
         /*
-         If message catalog for another architecture found (e.g. OpenMP RTL
-	 for IA-32 architecture opens libompui.dll for Intel(R) 64)
-	 Windows* OS returns error 193 (ERROR_BAD_EXE_FORMAT). However,
-         FormatMessage fails to return a message for this error, so user
-	 will see:
+          If message catalog for another architecture found (e.g. OpenMP RTL
+          for IA-32 architecture opens libompui.dll for Intel(R) 64)
+          Windows* OS returns error 193 (ERROR_BAD_EXE_FORMAT). However,
+          FormatMessage fails to return a message for this error, so user
+          will see:
 
-         OMP: Warning #2: Cannot open message catalog "1041\libompui.dll":
-         OMP: System error #193: (No system error message available)
-         OMP: Info #3: Default messages will be used.
+          OMP: Warning #2: Cannot open message catalog "1041\libompui.dll":
+          OMP: System error #193: (No system error message available)
+          OMP: Info #3: Default messages will be used.
 
-         Issue a hint in this case to let cause of trouble more understandable.
+          Issue a hint in this case to let cause of trouble more understandable.
         */
-	__kmp_msg(
-	    kmp_ms_warning,
-	    KMP_MSG( CantOpenMessageCatalog, path.str ),
-	    KMP_SYSERRCODE( error ),
+        kmp_msg_t err_code = KMP_SYSERRCODE(error);
+        __kmp_msg(
+            kmp_ms_warning,
+            KMP_MSG( CantOpenMessageCatalog, path.str ),
+            err_code,
             ( error == ERROR_BAD_EXE_FORMAT ? KMP_HNT( BadExeFormat, path.str, KMP_ARCH_STR ) : __kmp_msg_null ),
-	    __kmp_msg_null
-	);
-	KMP_INFORM( WillUseDefaultMessages );
+            __kmp_msg_null
+        );
+        if (__kmp_generate_warnings == kmp_warnings_off) {
+            __kmp_str_free(&err_code.str);
+        }
+
+        KMP_INFORM( WillUseDefaultMessages );
       }
     } else { // status == KMP_I18N_OPENED
 
@@ -703,7 +713,7 @@ __kmp_i18n_dump_catalog(
 
 kmp_msg_t
 __kmp_msg_format(
-    kmp_i18n_id_t id,
+    unsigned id_arg,
     ...
 ) {
 
@@ -712,7 +722,13 @@ __kmp_msg_format(
     kmp_str_buf_t  buffer;
     __kmp_str_buf_init( & buffer );
 
-    va_start( args, id );
+    va_start( args, id_arg );
+
+    // We use unsigned for the ID argument and explicitly cast it here to the
+    // right enumerator because variadic functions are not compatible with
+    // default promotions.
+    kmp_i18n_id_t id = (kmp_i18n_id_t)id_arg;
+
     #if KMP_OS_UNIX
         // On Linux* OS and OS X*, printf() family functions process parameter numbers, for example:
         // "%2$s %1$s".
@@ -803,7 +819,7 @@ sys_error(
                 int    strerror_r( int, char *, size_t );  // XSI version
         */
 
-        #if KMP_OS_LINUX
+        #if defined(__GLIBC__) && defined(_GNU_SOURCE)
 
             // GNU version of strerror_r.
 
@@ -818,9 +834,11 @@ sys_error(
             // XSI version of strerror_r.
 
             int    size   = 2048;
-            // TODO: Add checking result of malloc().
             char * buffer = (char *) KMP_INTERNAL_MALLOC( size );
             int    rc;
+            if (buffer == NULL) {
+                KMP_FATAL(MemoryAllocFailed);
+            }
             rc = strerror_r( err, buffer, size );
             if ( rc == -1 ) {
                 rc = errno;            // XSI version sets errno.
@@ -829,6 +847,9 @@ sys_error(
                 KMP_INTERNAL_FREE( buffer );
                 size *= 2;
                 buffer = (char *) KMP_INTERNAL_MALLOC( size );
+                if (buffer == NULL) {
+                    KMP_FATAL(MemoryAllocFailed);
+                }
                 rc = strerror_r( err, buffer, size );
                 if ( rc == -1 ) {
                     rc = errno;        // XSI version sets errno.
@@ -920,9 +941,9 @@ __kmp_msg(
         };
     }; // switch
     fmsg = __kmp_msg_format( format, message.num, message.str );
-    KMP_INTERNAL_FREE( (void *) message.str );
+    __kmp_str_free(&message.str);
     __kmp_str_buf_cat( & buffer, fmsg.str, fmsg.len );
-    KMP_INTERNAL_FREE( (void *) fmsg.str );
+    __kmp_str_free(&fmsg.str);
 
     // Format other messages.
     va_start( args, message );
@@ -946,9 +967,9 @@ __kmp_msg(
             };
         }; // switch
         fmsg = __kmp_msg_format( format, message.num, message.str );
-        KMP_INTERNAL_FREE( (void *) message.str );
+        __kmp_str_free(&message.str);
         __kmp_str_buf_cat( & buffer, fmsg.str, fmsg.len );
-        KMP_INTERNAL_FREE( (void *) fmsg.str );
+        __kmp_str_free(&fmsg.str);
     }; // forever
     va_end( args );
 

runtime/src/kmp_i18n.h

@@ -128,7 +128,7 @@ extern kmp_msg_t __kmp_msg_null;   // Denotes the end of variadic list of argume
 // Helper functions. Creates messages either from message catalog or from system. Note: these
 // functions allocate memory. You should pass created messages to __kmp_msg() function, it will
 // print messages and destroy them.
-kmp_msg_t  __kmp_msg_format( kmp_i18n_id_t id, ... );
+kmp_msg_t  __kmp_msg_format( unsigned id_arg, ... );
 kmp_msg_t  __kmp_msg_error_code( int code );
 kmp_msg_t  __kmp_msg_error_mesg( char const * mesg );
 

runtime/src/kmp_import.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_import.c
+ * kmp_import.cpp
  */
 
 

runtime/src/kmp_io.cpp

@@ -1,5 +1,5 @@
 /*
- * KMP_IO.c -- RTL IO
+ * kmp_io.cpp -- RTL IO
  */
 
 
@@ -43,7 +43,7 @@ kmp_bootstrap_lock_t __kmp_console_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_
 
 #if KMP_OS_WINDOWS
 
-    # ifdef KMP_DEBUG 
+    # ifdef KMP_DEBUG
     /* __kmp_stdout is used only for dev build */
     static HANDLE    __kmp_stdout = NULL;
     # endif
@@ -77,7 +77,7 @@ kmp_bootstrap_lock_t __kmp_console_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_
         /* wait until user presses return before closing window */
         /* TODO only close if a window was opened */
         if( __kmp_console_exists ) {
-            #ifdef KMP_DEBUG 
+            #ifdef KMP_DEBUG
             /* standard out is used only in dev build */
             __kmp_stdout = NULL;
             #endif
@@ -95,7 +95,7 @@ kmp_bootstrap_lock_t __kmp_console_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_
         __kmp_acquire_bootstrap_lock( &__kmp_console_lock );
 
         if( ! __kmp_console_exists ) {
-            #ifdef KMP_DEBUG 
+            #ifdef KMP_DEBUG
             /* standard out is used only in dev build */
             HANDLE ho;
             #endif

runtime/src/kmp_itt.cpp

@@ -2,7 +2,7 @@
 
 #if USE_ITT_BUILD
 /*
- * kmp_itt.c -- ITT Notify interface.
+ * kmp_itt.cpp -- ITT Notify interface.
  */
 
 
@@ -32,6 +32,9 @@
     __itt_domain* __kmp_itt_imbalance_domains[KMP_MAX_FRAME_DOMAINS];
     kmp_int32 __kmp_itt_region_team_size[KMP_MAX_FRAME_DOMAINS];
     __itt_domain * metadata_domain = NULL;
+    __itt_string_handle * string_handle_imbl = NULL;
+    __itt_string_handle * string_handle_loop = NULL;
+    __itt_string_handle * string_handle_sngl = NULL;
 
     #include "kmp_version.h"
     #include "kmp_i18n.h"
@@ -100,13 +103,21 @@ __itt_error_handler(
     switch ( err ) {
         case __itt_error_no_module : {
             char const * library = va_arg( args, char const * );
-            #if KMP_OS_WINDOWS
-                int sys_err = va_arg( args, int );
-                __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), KMP_SYSERRCODE( sys_err ), __kmp_msg_null );
-            #else
-                char const * sys_err = va_arg( args, char const * );
-                __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), KMP_SYSERRMESG( sys_err ), __kmp_msg_null );
-            #endif
+#if KMP_OS_WINDOWS
+            int sys_err = va_arg( args, int );
+            kmp_msg_t err_code = KMP_SYSERRCODE( sys_err );
+            __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), err_code, __kmp_msg_null );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+              __kmp_str_free(&err_code.str);
+            }
+#else
+            char const * sys_err = va_arg( args, char const * );
+            kmp_msg_t err_code = KMP_SYSERRMESG( sys_err );
+            __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), err_code, __kmp_msg_null );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+              __kmp_str_free(&err_code.str);
+            }
+#endif
         } break;
         case __itt_error_no_symbol : {
             char const * library = va_arg( args, char const * );
@@ -127,12 +138,20 @@ __itt_error_handler(
         case __itt_error_cant_read_env : {
             char const * var     = va_arg( args, char const * );
             int          sys_err = va_arg( args, int );
-            __kmp_msg( kmp_ms_warning, KMP_MSG( CantGetEnvVar, var ), KMP_ERR( sys_err ), __kmp_msg_null );
+            kmp_msg_t err_code = KMP_ERR( sys_err );
+            __kmp_msg( kmp_ms_warning, KMP_MSG( CantGetEnvVar, var ), err_code, __kmp_msg_null );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+              __kmp_str_free(&err_code.str);
+            }
         } break;
         case __itt_error_system : {
             char const * func    = va_arg( args, char const * );
             int          sys_err = va_arg( args, int );
-            __kmp_msg( kmp_ms_warning, KMP_MSG( IttFunctionError, func ), KMP_SYSERRCODE( sys_err ), __kmp_msg_null );
+            kmp_msg_t err_code = KMP_SYSERRCODE( sys_err );
+            __kmp_msg( kmp_ms_warning, KMP_MSG( IttFunctionError, func ), err_code, __kmp_msg_null );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+              __kmp_str_free(&err_code.str);
+            }
         } break;
         default : {
             KMP_WARNING( IttUnknownError, err );

runtime/src/kmp_itt.h

@@ -53,8 +53,8 @@ void __kmp_itt_destroy();
 //     __kmp_itt_xxxed()  function should be called after action.
 
 // --- Parallel region reporting ---
-__kmp_inline void __kmp_itt_region_forking(  int gtid, int team_size, int barriers, int serialized = 0 ); // Master only, before forking threads.
-__kmp_inline void __kmp_itt_region_joined(   int gtid, int serialized = 0 ); // Master only, after joining threads.
+__kmp_inline void __kmp_itt_region_forking(  int gtid, int team_size, int barriers ); // Master only, before forking threads.
+__kmp_inline void __kmp_itt_region_joined(   int gtid ); // Master only, after joining threads.
     // (*) Note: A thread may execute tasks after this point, though.
 
 // --- Frame reporting ---
@@ -264,6 +264,9 @@ __kmp_inline void __kmp_itt_stack_callee_leave(__itt_caller);
     extern __itt_domain* __kmp_itt_imbalance_domains[KMP_MAX_FRAME_DOMAINS];
     extern kmp_int32 __kmp_itt_region_team_size[KMP_MAX_FRAME_DOMAINS];
     extern __itt_domain * metadata_domain;
+    extern __itt_string_handle * string_handle_imbl;
+    extern __itt_string_handle * string_handle_loop;
+    extern __itt_string_handle * string_handle_sngl;
 
 #else
 

runtime/src/kmp_itt.inl

@@ -89,10 +89,10 @@ static kmp_bootstrap_lock_t  metadata_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( met
 // -------------------------------------------------------------------------------------------------
 
 LINKAGE void
-__kmp_itt_region_forking( int gtid, int team_size, int barriers, int serialized ) {
+__kmp_itt_region_forking( int gtid, int team_size, int barriers ) {
 #if USE_ITT_NOTIFY
     kmp_team_t *      team = __kmp_team_from_gtid( gtid );
-    if (team->t.t_active_level + serialized > 1)
+    if (team->t.t_active_level > 1)
     {
         // The frame notifications are only supported for the outermost teams.
         return;
@@ -176,8 +176,8 @@ __kmp_itt_region_forking( int gtid, int team_size, int barriers, int serialized
             }
         }
         KMP_ITT_DEBUG_LOCK();
-        KMP_ITT_DEBUG_PRINT( "[frm beg] gtid=%d, idx=%x, serialized:%d, loc:%p\n",
-                         gtid, loc->reserved_2, serialized, loc );
+        KMP_ITT_DEBUG_PRINT( "[frm beg] gtid=%d, idx=%x, loc:%p\n",
+                         gtid, loc->reserved_2, loc );
     }
 #endif
 } // __kmp_itt_region_forking
@@ -315,20 +315,21 @@ __kmp_itt_metadata_imbalance( int gtid, kmp_uint64 begin, kmp_uint64 end, kmp_ui
         if( metadata_domain == NULL) {
             __itt_suppress_push(__itt_suppress_memory_errors);
             metadata_domain = __itt_domain_create( "OMP Metadata" );
+            string_handle_imbl = __itt_string_handle_create( "omp_metadata_imbalance");
+            string_handle_loop = __itt_string_handle_create( "omp_metadata_loop");
+            string_handle_sngl = __itt_string_handle_create( "omp_metadata_single");
             __itt_suppress_pop();
         }
         __kmp_release_bootstrap_lock( & metadata_lock );
     }
 
-    __itt_string_handle * string_handle = __itt_string_handle_create( "omp_metadata_imbalance");
-
     kmp_uint64 imbalance_data[ 4 ];
     imbalance_data[ 0 ] = begin;
     imbalance_data[ 1 ] = end;
     imbalance_data[ 2 ] = imbalance;
     imbalance_data[ 3 ] = reduction;
 
-    __itt_metadata_add(metadata_domain, __itt_null, string_handle, __itt_metadata_u64, 4, imbalance_data);
+    __itt_metadata_add(metadata_domain, __itt_null, string_handle_imbl, __itt_metadata_u64, 4, imbalance_data);
 #endif
 } // __kmp_itt_metadata_imbalance
 
@@ -342,24 +343,39 @@ __kmp_itt_metadata_loop( ident_t * loc, kmp_uint64 sched_type, kmp_uint64 iterat
         if( metadata_domain == NULL) {
             __itt_suppress_push(__itt_suppress_memory_errors);
             metadata_domain = __itt_domain_create( "OMP Metadata" );
+            string_handle_imbl = __itt_string_handle_create( "omp_metadata_imbalance");
+            string_handle_loop = __itt_string_handle_create( "omp_metadata_loop");
+            string_handle_sngl = __itt_string_handle_create( "omp_metadata_single");
             __itt_suppress_pop();
         }
         __kmp_release_bootstrap_lock( & metadata_lock );
     }
 
-    __itt_string_handle * string_handle = __itt_string_handle_create( "omp_metadata_loop");
-    kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->psource, 1 );
+    // Parse line and column from psource string: ";file;func;line;col;;"
+    char * s_line;
+    char * s_col;
+    KMP_DEBUG_ASSERT(loc->psource);
+#ifdef __cplusplus
+    s_line = strchr((char*)loc->psource, ';');
+#else
+    s_line = strchr(loc->psource, ';');
+#endif
+    KMP_DEBUG_ASSERT(s_line);
+    s_line = strchr(s_line + 1, ';');   // 2-nd semicolon
+    KMP_DEBUG_ASSERT(s_line);
+    s_line = strchr(s_line + 1, ';');   // 3-rd semicolon
+    KMP_DEBUG_ASSERT(s_line);
+    s_col = strchr(s_line + 1, ';');    // 4-th semicolon
+    KMP_DEBUG_ASSERT(s_col);
 
     kmp_uint64 loop_data[ 5 ];
-    loop_data[ 0 ] = str_loc.line;
-    loop_data[ 1 ] = str_loc.col;
+    loop_data[ 0 ] = atoi(s_line + 1); // read line
+    loop_data[ 1 ] = atoi(s_col + 1);  // read column
     loop_data[ 2 ] = sched_type;
     loop_data[ 3 ] = iterations;
     loop_data[ 4 ] = chunk;
 
-    __kmp_str_loc_free( &str_loc );
-
-    __itt_metadata_add(metadata_domain, __itt_null, string_handle, __itt_metadata_u64, 5, loop_data);
+    __itt_metadata_add(metadata_domain, __itt_null, string_handle_loop, __itt_metadata_u64, 5, loop_data);
 #endif
 } // __kmp_itt_metadata_loop
 
@@ -373,12 +389,14 @@ __kmp_itt_metadata_single( ident_t * loc ) {
         if( metadata_domain == NULL) {
             __itt_suppress_push(__itt_suppress_memory_errors);
             metadata_domain = __itt_domain_create( "OMP Metadata" );
+            string_handle_imbl = __itt_string_handle_create( "omp_metadata_imbalance");
+            string_handle_loop = __itt_string_handle_create( "omp_metadata_loop");
+            string_handle_sngl = __itt_string_handle_create( "omp_metadata_single");
             __itt_suppress_pop();
         }
         __kmp_release_bootstrap_lock( & metadata_lock );
     }
 
-    __itt_string_handle * string_handle = __itt_string_handle_create( "omp_metadata_single");
     kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->psource, 1 );
     kmp_uint64 single_data[ 2 ];
     single_data[ 0 ] = str_loc.line;
@@ -386,7 +404,7 @@ __kmp_itt_metadata_single( ident_t * loc ) {
 
     __kmp_str_loc_free( &str_loc );
 
-    __itt_metadata_add(metadata_domain, __itt_null, string_handle, __itt_metadata_u64, 2, single_data);
+    __itt_metadata_add(metadata_domain, __itt_null, string_handle_sngl, __itt_metadata_u64, 2, single_data);
 #endif
 } // __kmp_itt_metadata_single
 
@@ -409,10 +427,10 @@ __kmp_itt_region_finished( int gtid ) {
 // -------------------------------------------------------------------------------------------------
 
 LINKAGE void
-__kmp_itt_region_joined( int gtid, int serialized ) {
+__kmp_itt_region_joined( int gtid ) {
 #if USE_ITT_NOTIFY
     kmp_team_t *      team = __kmp_team_from_gtid( gtid );
-    if (team->t.t_active_level + serialized > 1)
+    if (team->t.t_active_level > 1)
     {
         // The frame notifications are only supported for the outermost teams.
         return;
@@ -424,8 +442,8 @@ __kmp_itt_region_joined( int gtid, int serialized ) {
         if(frm < KMP_MAX_FRAME_DOMAINS) {
             KMP_ITT_DEBUG_LOCK();
             __itt_frame_end_v3(__kmp_itt_region_domains[frm], NULL);
-            KMP_ITT_DEBUG_PRINT( "[frm end] gtid=%d, idx=%x, serialized:%d, loc:%p\n",
-                         gtid, loc->reserved_2, serialized, loc );
+            KMP_ITT_DEBUG_PRINT( "[frm end] gtid=%d, idx=%x, loc:%p\n",
+                         gtid, loc->reserved_2, loc );
         }
     }
 #endif

runtime/src/kmp_lock.cpp

@@ -14,6 +14,7 @@
 
 
 #include <stddef.h>
+#include <atomic>
 
 #include "kmp.h"
 #include "kmp_itt.h"
@@ -21,7 +22,9 @@
 #include "kmp_lock.h"
 #include "kmp_io.h"
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#include "tsan_annotations.h"
+
+#if KMP_USE_FUTEX
 # include <unistd.h>
 # include <sys/syscall.h>
 // We should really include <futex.h>, but that causes compatibility problems on different
@@ -90,7 +93,7 @@ __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
     KMP_MB();
 
 #ifdef USE_LOCK_PROFILE
-    kmp_uint32 curr = TCR_4( lck->lk.poll );
+    kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
     if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
         __kmp_printf( "LOCK CONTENTION: %p\n", lck );
     /* else __kmp_printf( "." );*/
@@ -113,11 +116,11 @@ __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
         KMP_YIELD_SPIN( spins );
     }
 
+    kmp_backoff_t backoff = __kmp_spin_backoff_params;
     while ( ( lck->lk.poll != KMP_LOCK_FREE(tas) ) ||
       ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) ) {
-        //
-        // FIXME - use exponential backoff here
-        //
+
+        __kmp_spin_backoff(&backoff);
         if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
           __kmp_xproc ) ) {
             KMP_YIELD( TRUE );
@@ -133,7 +136,9 @@ __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
 int
 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
 {
-    return __kmp_acquire_tas_lock_timed_template( lck, gtid );
+  int retval = __kmp_acquire_tas_lock_timed_template( lck, gtid );
+  ANNOTATE_TAS_ACQUIRED(lck);
+  return retval;
 }
 
 static int
@@ -178,6 +183,7 @@ __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 
     KMP_FSYNC_RELEASING(lck);
+    ANNOTATE_TAS_RELEASED(lck);
     KMP_ST_REL32( &(lck->lk.poll), KMP_LOCK_FREE(tas) );
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 
@@ -253,6 +259,7 @@ __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
     }
     else {
         __kmp_acquire_tas_lock_timed_template( lck, gtid );
+        ANNOTATE_TAS_ACQUIRED(lck);
         lck->lk.depth_locked = 1;
         return KMP_LOCK_ACQUIRED_FIRST;
     }
@@ -362,7 +369,7 @@ __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
 }
 
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
 
 /* ------------------------------------------------------------------------ */
 /* futex locks */
@@ -392,7 +399,7 @@ __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
     KMP_MB();
 
 #ifdef USE_LOCK_PROFILE
-    kmp_uint32 curr = TCR_4( lck->lk.poll );
+    kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
     if ( ( curr != 0 ) && ( curr != gtid_code ) )
         __kmp_printf( "LOCK CONTENTION: %p\n", lck );
     /* else __kmp_printf( "." );*/
@@ -466,7 +473,9 @@ __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
 int
 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
 {
-   return __kmp_acquire_futex_lock_timed_template( lck, gtid );
+  int retval = __kmp_acquire_futex_lock_timed_template( lck, gtid );
+  ANNOTATE_FUTEX_ACQUIRED(lck);
+  return retval;
 }
 
 static int
@@ -486,7 +495,7 @@ __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
 int
 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
 {
-    if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1, futex) << 1 ) ) {
+    if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY((gtid+1) << 1, futex) ) ) {
         KMP_FSYNC_ACQUIRED( lck );
         return TRUE;
     }
@@ -513,6 +522,7 @@ __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
       lck, lck->lk.poll, gtid ) );
 
     KMP_FSYNC_RELEASING(lck);
+    ANNOTATE_FUTEX_RELEASED(lck);
 
     kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex) );
 
@@ -602,6 +612,7 @@ __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
     }
     else {
         __kmp_acquire_futex_lock_timed_template( lck, gtid );
+        ANNOTATE_FUTEX_ACQUIRED(lck);
         lck->lk.depth_locked = 1;
         return KMP_LOCK_ACQUIRED_FIRST;
     }
@@ -710,7 +721,7 @@ __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
     __kmp_destroy_nested_futex_lock( lck );
 }
 
-#endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
+#endif // KMP_USE_FUTEX
 
 
 /* ------------------------------------------------------------------------ */
@@ -719,61 +730,56 @@ __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
 static kmp_int32
 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
 {
-    return TCR_4( lck->lk.owner_id ) - 1;
+    return std::atomic_load_explicit( &lck->lk.owner_id, std::memory_order_relaxed ) - 1;
 }
 
 static inline bool
 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
 {
-    return lck->lk.depth_locked != -1;
+    return std::atomic_load_explicit( &lck->lk.depth_locked, std::memory_order_relaxed ) != -1;
 }
 
 static kmp_uint32
-__kmp_bakery_check(kmp_uint32 value, kmp_uint32 checker)
+__kmp_bakery_check( void *now_serving, kmp_uint32 my_ticket )
 {
-    register kmp_uint32 pause;
-
-    if (value == checker) {
-        return TRUE;
-    }
-    for (pause = checker - value; pause != 0; --pause);
-    return FALSE;
+    return std::atomic_load_explicit( (std::atomic<unsigned> *)now_serving, std::memory_order_acquire ) == my_ticket;
 }
 
 __forceinline static int
 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
-    kmp_uint32 my_ticket;
-    KMP_MB();
-
-    my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
+    kmp_uint32 my_ticket = std::atomic_fetch_add_explicit( &lck->lk.next_ticket, 1U, std::memory_order_relaxed );
 
 #ifdef USE_LOCK_PROFILE
-    if ( TCR_4( lck->lk.now_serving ) != my_ticket )
+    if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) != my_ticket )
         __kmp_printf( "LOCK CONTENTION: %p\n", lck );
     /* else __kmp_printf( "." );*/
 #endif /* USE_LOCK_PROFILE */
 
-    if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
-        KMP_FSYNC_ACQUIRED(lck);
+    if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_acquire ) == my_ticket ) {
         return KMP_LOCK_ACQUIRED_FIRST;
     }
-    KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
-    KMP_FSYNC_ACQUIRED(lck);
+    KMP_WAIT_YIELD_PTR( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
     return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int
 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
-    return __kmp_acquire_ticket_lock_timed_template( lck, gtid );
+  int retval = __kmp_acquire_ticket_lock_timed_template( lck, gtid );
+  ANNOTATE_TICKET_ACQUIRED(lck);
+  return retval;
 }
 
 static int
 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     char const * const func = "omp_set_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -785,19 +791,19 @@ __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 
     __kmp_acquire_ticket_lock( lck, gtid );
 
-    lck->lk.owner_id = gtid + 1;
+    std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
     return KMP_LOCK_ACQUIRED_FIRST;
 }
 
 int
 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
-    kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
-    if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
+    kmp_uint32 my_ticket = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed );
+
+    if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) == my_ticket ) {
         kmp_uint32 next_ticket = my_ticket + 1;
-        if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
-          my_ticket, next_ticket ) ) {
-            KMP_FSYNC_ACQUIRED( lck );
+        if ( std::atomic_compare_exchange_strong_explicit( &lck->lk.next_ticket,
+             &my_ticket, next_ticket, std::memory_order_acquire, std::memory_order_acquire )) {
             return TRUE;
         }
     }
@@ -808,7 +814,11 @@ static int
 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     char const * const func = "omp_test_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -818,7 +828,7 @@ __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
     int retval = __kmp_test_ticket_lock( lck, gtid );
 
     if ( retval ) {
-        lck->lk.owner_id = gtid + 1;
+        std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
     }
     return retval;
 }
@@ -826,16 +836,10 @@ __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 int
 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
-    kmp_uint32  distance;
+    kmp_uint32 distance = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed ) - std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed );
 
-    KMP_MB();       /* Flush all pending memory write invalidates.  */
-
-    KMP_FSYNC_RELEASING(lck);
-    distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
-
-    KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
-
-    KMP_MB();       /* Flush all pending memory write invalidates.  */
+    ANNOTATE_TICKET_RELEASED(lck);
+    std::atomic_fetch_add_explicit( &lck->lk.now_serving, 1U, std::memory_order_release );
 
     KMP_YIELD( distance
       > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
@@ -846,8 +850,11 @@ static int
 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     char const * const func = "omp_unset_lock";
-    KMP_MB();  /* in case another processor initialized lock */
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -860,7 +867,7 @@ __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
       && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
         KMP_FATAL( LockUnsettingSetByAnother, func );
     }
-    lck->lk.owner_id = 0;
+    std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
     return __kmp_release_ticket_lock( lck, gtid );
 }
 
@@ -868,11 +875,12 @@ void
 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
 {
     lck->lk.location = NULL;
-    TCW_4( lck->lk.next_ticket, 0 );
-    TCW_4( lck->lk.now_serving, 0 );
-    lck->lk.owner_id = 0;      // no thread owns the lock.
-    lck->lk.depth_locked = -1; // -1 => not a nested lock.
-    lck->lk.initialized = (kmp_ticket_lock *)lck;
+    lck->lk.self = lck;
+    std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
+    std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
+    std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed ); // no thread owns the lock.
+    std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed ); // -1 => not a nested lock.
+    std::atomic_store_explicit( &lck->lk.initialized, true, std::memory_order_release );
 }
 
 static void
@@ -884,19 +892,24 @@ __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
 void
 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
 {
-    lck->lk.initialized = NULL;
-    lck->lk.location    = NULL;
-    lck->lk.next_ticket = 0;
-    lck->lk.now_serving = 0;
-    lck->lk.owner_id = 0;
-    lck->lk.depth_locked = -1;
+    std::atomic_store_explicit( &lck->lk.initialized, false, std::memory_order_release );
+    lck->lk.self = NULL;
+    lck->lk.location = NULL;
+    std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
+    std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
+    std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
+    std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed );
 }
 
 static void
 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
 {
     char const * const func = "omp_destroy_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -919,15 +932,14 @@ __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
     KMP_DEBUG_ASSERT( gtid >= 0 );
 
     if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
-        lck->lk.depth_locked += 1;
+        std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
         return KMP_LOCK_ACQUIRED_NEXT;
     }
     else {
         __kmp_acquire_ticket_lock_timed_template( lck, gtid );
-        KMP_MB();
-        lck->lk.depth_locked = 1;
-        KMP_MB();
-        lck->lk.owner_id = gtid + 1;
+        ANNOTATE_TICKET_ACQUIRED(lck);
+        std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
+        std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
         return KMP_LOCK_ACQUIRED_FIRST;
     }
 }
@@ -936,7 +948,11 @@ static int
 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     char const * const func = "omp_set_nest_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -953,16 +969,15 @@ __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
     KMP_DEBUG_ASSERT( gtid >= 0 );
 
     if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
-        retval = ++lck->lk.depth_locked;
+        retval = std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed ) + 1;
     }
     else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
         retval = 0;
     }
     else {
-        KMP_MB();
-        retval = lck->lk.depth_locked = 1;
-        KMP_MB();
-        lck->lk.owner_id = gtid + 1;
+        std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
+        std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
+        retval = 1;
     }
     return retval;
 }
@@ -972,7 +987,11 @@ __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
   kmp_int32 gtid )
 {
     char const * const func = "omp_test_nest_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -986,10 +1005,8 @@ __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     KMP_DEBUG_ASSERT( gtid >= 0 );
 
-    KMP_MB();
-    if ( --(lck->lk.depth_locked) == 0 ) {
-        KMP_MB();
-        lck->lk.owner_id = 0;
+    if ( ( std::atomic_fetch_add_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed ) - 1 ) == 0 ) {
+        std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
         __kmp_release_ticket_lock( lck, gtid );
         return KMP_LOCK_RELEASED;
     }
@@ -1000,8 +1017,11 @@ static int
 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
 {
     char const * const func = "omp_unset_nest_lock";
-    KMP_MB();  /* in case another processor initialized lock */
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -1020,7 +1040,7 @@ void
 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
 {
     __kmp_init_ticket_lock( lck );
-    lck->lk.depth_locked = 0; // >= 0 for nestable locks, -1 for simple locks
+    std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed ); // >= 0 for nestable locks, -1 for simple locks
 }
 
 static void
@@ -1033,14 +1053,18 @@ void
 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
 {
     __kmp_destroy_ticket_lock( lck );
-    lck->lk.depth_locked = 0;
+    std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed );
 }
 
 static void
 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
 {
     char const * const func = "omp_destroy_nest_lock";
-    if ( lck->lk.initialized != lck ) {
+
+    if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
+        KMP_FATAL( LockIsUninitialized, func );
+    }
+    if ( lck->lk.self != lck ) {
         KMP_FATAL( LockIsUninitialized, func );
     }
     if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
@@ -1060,7 +1084,7 @@ __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
 static int
 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
 {
-    return lck == lck->lk.initialized;
+    return std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) && ( lck->lk.self == lck);
 }
 
 static const ident_t *
@@ -1408,7 +1432,9 @@ __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
 {
     KMP_DEBUG_ASSERT( gtid >= 0 );
 
-    return __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
+    int retval = __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
+    ANNOTATE_QUEUING_ACQUIRED(lck);
+    return retval;
 }
 
 static int
@@ -1458,6 +1484,7 @@ __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
         if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
             KA_TRACE( 1000, ("__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
             KMP_FSYNC_ACQUIRED(lck);
+            ANNOTATE_QUEUING_ACQUIRED(lck);
             return TRUE;
         }
     }
@@ -1508,6 +1535,7 @@ __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
     KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
 
     KMP_FSYNC_RELEASING(lck);
+    ANNOTATE_QUEUING_RELEASED(lck);
 
     while( 1 ) {
         kmp_int32 dequeued;
@@ -1712,6 +1740,7 @@ __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
     }
     else {
         __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
+        ANNOTATE_QUEUING_ACQUIRED(lck);
         KMP_MB();
         lck->lk.depth_locked = 1;
         KMP_MB();
@@ -2360,6 +2389,7 @@ __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
     __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
     // We have acquired the base lock, so count that.
     KMP_INC_STAT(lck,nonSpeculativeAcquires );
+    ANNOTATE_QUEUING_ACQUIRED(lck);
 }
 
 static void
@@ -2498,7 +2528,7 @@ __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
     // polling area has been reconfigured.  Unless it is reconfigured, the
     // reloads stay in L1 cache and are cheap.
     //
-    // Keep this code in sync with KMP_WAIT_YIELD, in kmp_dispatch.c !!!
+    // Keep this code in sync with KMP_WAIT_YIELD, in kmp_dispatch.cpp !!!
     //
     // The current implementation of KMP_WAIT_YIELD doesn't allow for mask
     // and poll to be re-read every spin iteration.
@@ -2507,7 +2537,7 @@ __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
 
     KMP_FSYNC_PREPARE(lck);
     KMP_INIT_YIELD(spins);
-    while (TCR_8(polls[ticket & mask]).poll < ticket) { // volatile load
+    while (TCR_8(polls[ticket & mask].poll) < ticket) { // volatile load
         // If we are oversubscribed,
         // or have waited a bit (and KMP_LIBRARY=turnaround), then yield.
         // CPU Pause is in the macros for yield.
@@ -2647,7 +2677,9 @@ __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
 int
 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
 {
-    return __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
+    int retval = __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
+    ANNOTATE_DRDPA_ACQUIRED(lck);
+    return retval;
 }
 
 static int
@@ -2741,6 +2773,7 @@ __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
     KA_TRACE(1000, ("__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
        ticket - 1, lck));
     KMP_FSYNC_RELEASING(lck);
+    ANNOTATE_DRDPA_RELEASED(lck);
     KMP_ST_REL64(&(polls[ticket & mask].poll), ticket); // volatile store
     return KMP_LOCK_RELEASED;
 }
@@ -2846,6 +2879,7 @@ __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
     }
     else {
         __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
+        ANNOTATE_DRDPA_ACQUIRED(lck);
         KMP_MB();
         lck->lk.depth_locked = 1;
         KMP_MB();
@@ -3008,6 +3042,46 @@ __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
     lck->lk.flags = flags;
 }
 
+// Time stamp counter
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+# define __kmp_tsc() __kmp_hardware_timestamp()
+// Runtime's default backoff parameters
+kmp_backoff_t __kmp_spin_backoff_params = { 1, 4096, 100 };
+#else
+// Use nanoseconds for other platforms
+extern kmp_uint64 __kmp_now_nsec();
+kmp_backoff_t __kmp_spin_backoff_params = { 1, 256, 100 };
+# define __kmp_tsc() __kmp_now_nsec()
+#endif
+
+// A useful predicate for dealing with timestamps that may wrap.
+// Is a before b?
+// Since the timestamps may wrap, this is asking whether it's
+// shorter to go clockwise from a to b around the clock-face, or anti-clockwise.
+// Times where going clockwise is less distance than going anti-clockwise
+// are in the future, others are in the past.
+// e.g.) a = MAX-1, b = MAX+1 (=0), then a > b (true) does not mean a reached b
+//       whereas signed(a) = -2, signed(b) = 0 captures the actual difference
+static inline bool before(kmp_uint64 a, kmp_uint64 b)
+{
+    return ((kmp_int64)b - (kmp_int64)a) > 0;
+}
+
+// Truncated binary exponential backoff function
+void
+__kmp_spin_backoff(kmp_backoff_t *boff)
+{
+    // We could flatten this loop, but making it a nested loop gives better result.
+    kmp_uint32 i;
+    for (i = boff->step; i > 0; i--) {
+        kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
+        do {
+            KMP_CPU_PAUSE();
+        } while (before(__kmp_tsc(), goal));
+    }
+    boff->step = (boff->step<<1 | 1) & (boff->max_backoff-1);
+}
+
 #if KMP_USE_DYNAMIC_LOCK
 
 // Direct lock initializers. It simply writes a tag to the low 8 bits of the lock word.
@@ -3354,7 +3428,7 @@ __kmp_lookup_indirect_lock(void **user_lock, const char *func)
         if (lck == NULL) {
             KMP_FATAL(LockIsUninitialized, func);
         }
-        return lck; 
+        return lck;
     } else {
         if (OMP_LOCK_T_SIZE < sizeof(void *)) {
             return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
@@ -3448,7 +3522,7 @@ __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
 
 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
 
-// This is used only in kmp_error.c when consistency checking is on.
+// This is used only in kmp_error.cpp when consistency checking is on.
 kmp_int32
 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
 {
@@ -3456,7 +3530,7 @@ __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
         case lockseq_tas:
         case lockseq_nested_tas:
             return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
-#if KMP_HAS_FUTEX
+#if KMP_USE_FUTEX
         case lockseq_futex:
         case lockseq_nested_futex:
             return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
@@ -3468,8 +3542,8 @@ __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
         case lockseq_nested_queuing:
 #if KMP_USE_ADAPTIVE_LOCKS
         case lockseq_adaptive:
-            return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
 #endif
+            return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
         case lockseq_drdpa:
         case lockseq_nested_drdpa:
             return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
@@ -3499,12 +3573,18 @@ __kmp_init_dynamic_user_locks()
         __kmp_indirect_unset = indirect_unset;
         __kmp_indirect_test  = indirect_test;
     }
+    // If the user locks have already been initialized, then return.
+    // Allow the switch between different KMP_CONSISTENCY_CHECK values,
+    // but do not allocate new lock tables if they have already been
+    // allocated.
+    if (__kmp_init_user_locks)
+        return;
 
     // Initialize lock index table
     __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
     __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(sizeof(kmp_indirect_lock_t *));
     *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)
-                                  __kmp_allocate(KMP_I_LOCK_CHUNK*sizeof(kmp_indirect_lock_t)); 
+                                  __kmp_allocate(KMP_I_LOCK_CHUNK*sizeof(kmp_indirect_lock_t));
     __kmp_i_lock_table.next = 0;
 
     // Indirect lock size
@@ -3578,6 +3658,7 @@ __kmp_cleanup_indirect_user_locks()
             __kmp_free(ll->lock);
             ll->lock = NULL;
         }
+        __kmp_indirect_lock_pool[k] = NULL;
     }
     // Clean up the remaining undestroyed locks.
     for (i = 0; i < __kmp_i_lock_table.next; i++) {
@@ -3680,7 +3761,7 @@ void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
         }
         break;
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
+#if KMP_USE_FUTEX
 
         case lk_futex: {
             __kmp_base_user_lock_size = sizeof( kmp_base_futex_lock_t );
@@ -3720,7 +3801,7 @@ void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
         }
         break;
 
-#endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
+#endif // KMP_USE_FUTEX
 
         case lk_ticket: {
             __kmp_base_user_lock_size = sizeof( kmp_base_ticket_lock_t );
@@ -3983,12 +4064,17 @@ __kmp_user_lock_allocate( void **user_lock, kmp_int32 gtid,
 
     if ( __kmp_lock_pool == NULL ) {
         // Lock pool is empty. Allocate new memory.
+
+        // ANNOTATION: Found no good way to express the syncronisation
+        // between allocation and usage, so ignore the allocation
+        ANNOTATE_IGNORE_WRITES_BEGIN();
         if ( __kmp_num_locks_in_block <= 1 ) { // Tune this cutoff point.
             lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
         }
         else {
             lck = __kmp_lock_block_allocate();
         }
+        ANNOTATE_IGNORE_WRITES_END();
 
         // Insert lock in the table so that it can be freed in __kmp_cleanup,
         // and debugger has info on all allocated locks.

runtime/src/kmp_lock.h

@@ -23,6 +23,8 @@
 #include "kmp_debug.h"
 
 #ifdef __cplusplus
+#include <atomic>
+
 extern "C" {
 #endif // __cplusplus
 
@@ -176,7 +178,9 @@ extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
 #define KMP_LOCK_ACQUIRED_FIRST 1
 #define KMP_LOCK_ACQUIRED_NEXT  0
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#define KMP_USE_FUTEX (KMP_OS_LINUX && !KMP_OS_CNK && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
+
+#if KMP_USE_FUTEX
 
 // ----------------------------------------------------------------------------
 // futex locks.  futex locks are only available on Linux* OS.
@@ -226,23 +230,48 @@ extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gti
 extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
 extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
 
-#endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#endif // KMP_USE_FUTEX
 
 
 // ----------------------------------------------------------------------------
 // Ticket locks.
 // ----------------------------------------------------------------------------
 
+#ifdef __cplusplus
+
+#ifdef _MSC_VER
+// MSVC won't allow use of std::atomic<> in a union since it has non-trivial copy constructor.
+
 struct kmp_base_ticket_lock {
     // `initialized' must be the first entry in the lock data structure!
-    volatile union kmp_ticket_lock * initialized;  // points to the lock union if in initialized state
-    ident_t const *     location;     // Source code location of omp_init_lock().
-    volatile kmp_uint32 next_ticket;  // ticket number to give to next thread which acquires
-    volatile kmp_uint32 now_serving;  // ticket number for thread which holds the lock
-    volatile kmp_int32  owner_id;     // (gtid+1) of owning thread, 0 if unlocked
-    kmp_int32           depth_locked; // depth locked, for nested locks only
-    kmp_lock_flags_t    flags;        // lock specifics, e.g. critical section lock
+    std::atomic_bool      initialized;
+    volatile union kmp_ticket_lock *self; // points to the lock union
+    ident_t const *       location;       // Source code location of omp_init_lock().
+    std::atomic_uint      next_ticket;    // ticket number to give to next thread which acquires
+    std::atomic_uint      now_serving;    // ticket number for thread which holds the lock
+    std::atomic_int       owner_id;       // (gtid+1) of owning thread, 0 if unlocked
+    std::atomic_int       depth_locked;   // depth locked, for nested locks only
+    kmp_lock_flags_t      flags;          // lock specifics, e.g. critical section lock
 };
+#else
+struct kmp_base_ticket_lock {
+    // `initialized' must be the first entry in the lock data structure!
+    std::atomic<bool>     initialized;
+    volatile union kmp_ticket_lock *self; // points to the lock union
+    ident_t const *       location;       // Source code location of omp_init_lock().
+    std::atomic<unsigned> next_ticket;    // ticket number to give to next thread which acquires
+    std::atomic<unsigned> now_serving;    // ticket number for thread which holds the lock
+    std::atomic<int>      owner_id;       // (gtid+1) of owning thread, 0 if unlocked
+    std::atomic<int>      depth_locked;   // depth locked, for nested locks only
+    kmp_lock_flags_t      flags;          // lock specifics, e.g. critical section lock
+};
+#endif
+
+#else // __cplusplus
+
+struct kmp_base_ticket_lock;
+
+#endif // !__cplusplus
 
 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
 
@@ -260,7 +289,13 @@ typedef union kmp_ticket_lock kmp_ticket_lock_t;
 //    kmp_ticket_lock_t xlock = KMP_TICKET_LOCK_INITIALIZER( xlock );
 // Note the macro argument. It is important to make var properly initialized.
 //
-#define KMP_TICKET_LOCK_INITIALIZER( lock ) { { (kmp_ticket_lock_t *) & (lock), NULL, 0, 0, 0, -1 } }
+#define KMP_TICKET_LOCK_INITIALIZER( lock ) { { ATOMIC_VAR_INIT(true), \
+                                                              &(lock), \
+                                                                 NULL, \
+                                                  ATOMIC_VAR_INIT(0U), \
+                                                  ATOMIC_VAR_INIT(0U), \
+                                                   ATOMIC_VAR_INIT(0), \
+                                                  ATOMIC_VAR_INIT(-1)    } }
 
 extern int __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
 extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
@@ -592,7 +627,7 @@ __kmp_destroy_lock( kmp_lock_t *lck )
 enum kmp_lock_kind {
     lk_default = 0,
     lk_tas,
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     lk_futex,
 #endif
 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
@@ -613,7 +648,7 @@ extern kmp_lock_kind_t __kmp_user_lock_kind;
 
 union kmp_user_lock {
     kmp_tas_lock_t     tas;
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+#if KMP_USE_FUTEX
     kmp_futex_lock_t   futex;
 #endif
     kmp_ticket_lock_t  ticket;
@@ -1038,7 +1073,7 @@ extern void __kmp_cleanup_user_locks();
 // KMP_USE_DYNAMIC_LOCK enables dynamic dispatch of lock functions without breaking the current
 // compatibility. Essential functionality of this new code is dynamic dispatch, but it also
 // implements (or enables implementation of) hinted user lock and critical section which will be
-// part of OMP 4.1 soon.
+// part of OMP 4.5 soon.
 //
 // Lock type can be decided at creation time (i.e., lock initialization), and subsequent lock
 // function call on the created lock object requires type extraction and call through jump table
@@ -1068,9 +1103,8 @@ extern void __kmp_cleanup_user_locks();
 #include <stdint.h> // for uintptr_t
 
 // Shortcuts
-#define KMP_USE_FUTEX          (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
-#define KMP_USE_INLINED_TAS    (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1
-#define KMP_USE_INLINED_FUTEX  KMP_USE_FUTEX && 0
+#define KMP_USE_INLINED_TAS   (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1
+#define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0
 
 // List of lock definitions; all nested locks are indirect locks.
 // hle lock is xchg lock prefixed with XACQUIRE/XRELEASE.
@@ -1200,7 +1234,7 @@ extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(void **, kmp_int32, km
 // Cleans up global states and data structures for managing dynamic user locks.
 extern void __kmp_cleanup_indirect_user_locks();
 
-// Default user lock sequence when not using hinted locks. 
+// Default user lock sequence when not using hinted locks.
 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
 
 // Jump table for "set lock location", available only for indirect locks.
@@ -1253,7 +1287,7 @@ extern int __kmp_num_locks_in_block;
                                ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \
                                : *((kmp_indirect_lock_t **)(l)) )
 
-// Used once in kmp_error.c
+// Used once in kmp_error.cpp
 extern kmp_int32
 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
 
@@ -1265,6 +1299,19 @@ __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
 
 #endif // KMP_USE_DYNAMIC_LOCK
 
+// data structure for using backoff within spin locks.
+typedef struct {
+    kmp_uint32 step;        // current step
+    kmp_uint32 max_backoff; // upper bound of outer delay loop
+    kmp_uint32 min_tick;    // size of inner delay loop in ticks (machine-dependent)
+} kmp_backoff_t;
+
+// Runtime's default backoff parameters
+extern kmp_backoff_t __kmp_spin_backoff_params;
+
+// Backoff function
+extern void __kmp_spin_backoff(kmp_backoff_t *);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif // __cplusplus

runtime/src/kmp_omp.h

@@ -218,6 +218,7 @@ typedef struct {
     /* kmp_thread_data_t */
     kmp_int32          hd_sizeof_struct;
     offset_and_size_t  hd_deque;
+    offset_and_size_t  hd_deque_size;
     offset_and_size_t  hd_deque_head;
     offset_and_size_t  hd_deque_tail;
     offset_and_size_t  hd_deque_ntasks;

runtime/src/kmp_os.h

@@ -147,9 +147,9 @@
 # define KMP_UINT64_SPEC     "llu"
 #endif /* KMP_OS_UNIX */
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS
 # define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
-#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
+#elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
 # define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 # error "Can't determine size_t printf format specifier."
@@ -236,9 +236,18 @@ typedef double  kmp_real64;
 #endif
 
 #define PAGE_SIZE                       (0x4000)
+
+#if KMP_OS_LINUX
+#define KMP_GET_PAGE_SIZE() getpagesize()
+#else
+// TODO: find the corresponding function to getpagesize() in Windows
+// and use it whenever possible.
+#define KMP_GET_PAGE_SIZE() PAGE_SIZE
+#endif
+
 #define PAGE_ALIGNED(_addr)     ( ! ((size_t) _addr & \
-                                     (size_t)(PAGE_SIZE - 1)))
-#define ALIGN_TO_PAGE(x)   (void *)(((size_t)(x)) & ~((size_t)(PAGE_SIZE - 1)))
+                                     (size_t)(KMP_GET_PAGE_SIZE() - 1)))
+#define ALIGN_TO_PAGE(x)   (void *)(((size_t)(x)) & ~((size_t)(KMP_GET_PAGE_SIZE() - 1)))
 
 /* ---------------------- Support for cache alignment, padding, etc. -----------------*/
 
@@ -550,7 +559,7 @@ extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
 # endif
 #endif /* KMP_OS_WINDOWS */
 
-#if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64
+#if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64
 # define KMP_MB()       __sync_synchronize()
 #endif
 
@@ -596,8 +605,12 @@ extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
 
 #define TCR_4(a)            (a)
 #define TCW_4(a,b)          (a) = (b)
+#define TCI_4(a)            (++(a))
+#define TCD_4(a)            (--(a))
 #define TCR_8(a)            (a)
 #define TCW_8(a,b)          (a) = (b)
+#define TCI_8(a)            (++(a))
+#define TCD_8(a)            (--(a))
 #define TCR_SYNC_4(a)       (a)
 #define TCW_SYNC_4(a,b)     (a) = (b)
 #define TCX_SYNC_4(a,b,c)   KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), (kmp_int32)(b), (kmp_int32)(c))
@@ -605,7 +618,7 @@ extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
 #define TCW_SYNC_8(a,b)     (a) = (b)
 #define TCX_SYNC_8(a,b,c)   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))
 
-#if KMP_ARCH_X86
+#if KMP_ARCH_X86 || KMP_ARCH_MIPS
 // What about ARM?
     #define TCR_PTR(a)          ((void *)TCR_4(a))
     #define TCW_PTR(a,b)        TCW_4((a),(b))
@@ -646,6 +659,7 @@ typedef void    (*microtask_t)( int *gtid, int *npr, ... );
 #endif
 
 #define KMP_WAIT_YIELD           __kmp_wait_yield_4
+#define KMP_WAIT_YIELD_PTR       __kmp_wait_yield_4_ptr
 #define KMP_EQ                   __kmp_eq_4
 #define KMP_NEQ                  __kmp_neq_4
 #define KMP_LT                   __kmp_lt_4
@@ -674,7 +688,7 @@ typedef void    (*microtask_t)( int *gtid, int *npr, ... );
 #endif
 
 // Enable dynamic user lock
-#if OMP_41_ENABLED
+#if OMP_45_ENABLED
 # define KMP_USE_DYNAMIC_LOCK 1
 #endif
 

runtime/src/kmp_platform.h

@@ -77,6 +77,8 @@
 #define KMP_ARCH_PPC64_BE   0
 #define KMP_ARCH_PPC64_LE   0
 #define KMP_ARCH_PPC64 (KMP_ARCH_PPC64_LE || KMP_ARCH_PPC64_BE)
+#define KMP_ARCH_MIPS        0
+#define KMP_ARCH_MIPS64      0
 
 #if KMP_OS_WINDOWS
 # if defined _M_AMD64
@@ -106,6 +108,14 @@
 # elif defined __aarch64__
 #  undef KMP_ARCH_AARCH64
 #  define KMP_ARCH_AARCH64 1
+# elif defined __mips__
+#  if defined __mips64
+#   undef KMP_ARCH_MIPS64
+#   define KMP_ARCH_MIPS64 1
+#  else
+#   undef KMP_ARCH_MIPS
+#   define KMP_ARCH_MIPS 1
+#  endif
 # endif
 #endif
 
@@ -161,10 +171,10 @@
 #endif
 
 /* Specify 32 bit architectures here */
-#define KMP_32_BIT_ARCH (KMP_ARCH_X86 || KMP_ARCH_ARM)
+#define KMP_32_BIT_ARCH (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS)
 
 // TODO: Fixme - This is clever, but really fugly
-#if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 + KMP_ARCH_AARCH64)
+#if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 + KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64)
 # error Unknown or unsupported architecture
 #endif
 

runtime/src/kmp_safe_c_api.h

@@ -50,7 +50,7 @@
 # define KMP_MEMCPY_S(dst, bsz, src, cnt)   memcpy(dst, src, cnt)
 # define KMP_SNPRINTF                       snprintf
 # define KMP_SSCANF                         sscanf
-# define KMP_STRCPY_S(dst, bsz, src)        strcpy(dst, src) 
+# define KMP_STRCPY_S(dst, bsz, src)        strcpy(dst, src)
 # define KMP_STRNCPY_S(dst, bsz, src, cnt)  strncpy(dst, src, cnt)
 # define KMP_VSNPRINTF                      vsnprintf
 # define KMP_STRNCPY                        strncpy

runtime/src/kmp_sched.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_sched.c -- static scheduling -- iteration initialization
+ * kmp_sched.cpp -- static scheduling -- iteration initialization
  */
 
 
@@ -84,7 +84,7 @@ __kmp_for_static_init(
     typename traits_t< T >::signed_t  chunk
 ) {
     KMP_COUNT_BLOCK(OMP_FOR_static);
-    KMP_TIME_BLOCK (FOR_static_scheduling);
+    KMP_TIME_PARTITIONED_BLOCK(FOR_static_scheduling);
 
     typedef typename traits_t< T >::unsigned_t  UT;
     typedef typename traits_t< T >::signed_t    ST;
@@ -97,8 +97,8 @@ __kmp_for_static_init(
     register kmp_info_t *th = __kmp_threads[ gtid ];
 
 #if OMPT_SUPPORT && OMPT_TRACE
-    ompt_team_info_t *team_info = NULL; 
-    ompt_task_info_t *task_info = NULL; 
+    ompt_team_info_t *team_info = NULL;
+    ompt_task_info_t *task_info = NULL;
 
     if (ompt_enabled) {
         // Only fully initialize variables needed by OMPT if OMPT is enabled.
@@ -164,6 +164,9 @@ __kmp_for_static_init(
     }
 
     #if OMP_40_ENABLED
+    // Although there are schedule enumerations above kmp_ord_upper which are not schedules for "distribute",
+    // the only ones which are useful are dynamic, so cannot be seen here, since this codepath is only executed
+    // for static schedules.
     if ( schedtype > kmp_ord_upper ) {
         // we are in DISTRIBUTE construct
         schedtype += kmp_sch_static - kmp_distribute_static;      // AC: convert to usual schedule type
@@ -241,12 +244,11 @@ __kmp_for_static_init(
         trip_count = *pupper - *plower + 1;
     } else if (incr == -1) {
         trip_count = *plower - *pupper + 1;
+    } else if ( incr > 0 ) {
+        // upper-lower can exceed the limit of signed type
+        trip_count = (UT)(*pupper - *plower) / incr + 1;
     } else {
-        if ( incr > 1 ) {  // the check is needed for unsigned division when incr < 0
-            trip_count = (*pupper - *plower) / incr + 1;
-        } else {
-            trip_count = (*plower - *pupper) / ( -incr ) + 1;
-        }
+        trip_count = (UT)(*plower - *pupper) / (-incr) + 1;
     }
 
     if ( __kmp_env_consistency_check ) {
@@ -322,6 +324,29 @@ __kmp_for_static_init(
                 *plastiter = (tid == ((trip_count - 1)/( UT )chunk) % nth);
             break;
         }
+#if OMP_45_ENABLED
+    case kmp_sch_static_balanced_chunked:
+        {
+            register T old_upper = *pupper;
+            // round up to make sure the chunk is enough to cover all iterations
+            register UT span = (trip_count+nth-1) / nth;
+
+            // perform chunk adjustment
+            chunk = (span + chunk - 1) & ~(chunk-1);
+
+            span = chunk * incr;
+            *plower = *plower + (span * tid);
+            *pupper = *plower + span - incr;
+            if ( incr > 0 ) {
+              if ( *pupper > old_upper ) *pupper = old_upper;
+            } else
+              if ( *pupper < old_upper ) *pupper = old_upper;
+
+            if( plastiter != NULL )
+                *plastiter = ( tid == ((trip_count - 1)/( UT )chunk) );
+            break;
+        }
+#endif
     default:
         KMP_ASSERT2( 0, "__kmpc_for_static_init: unknown scheduling type" );
         break;
@@ -444,8 +469,11 @@ __kmp_dist_for_static_init(
         trip_count = *pupper - *plower + 1;
     } else if(incr == -1) {
         trip_count = *plower - *pupper + 1;
+    } else if ( incr > 0 ) {
+        // upper-lower can exceed the limit of signed type
+        trip_count = (UT)(*pupper - *plower) / incr + 1;
     } else {
-        trip_count = (ST)(*pupper - *plower) / incr + 1; // cast to signed to cover incr<0 case
+        trip_count = (UT)(*plower - *pupper) / (-incr) + 1;
     }
 
     *pstride = *pupper - *plower;  // just in case (can be unused)
@@ -511,8 +539,11 @@ __kmp_dist_for_static_init(
             trip_count = *pupperDist - *plower + 1;
         } else if(incr == -1) {
             trip_count = *plower - *pupperDist + 1;
+        } else if ( incr > 1 ) {
+            // upper-lower can exceed the limit of signed type
+            trip_count = (UT)(*pupperDist - *plower) / incr + 1;
         } else {
-            trip_count = (ST)(*pupperDist - *plower) / incr + 1;
+            trip_count = (UT)(*plower - *pupperDist) / (-incr) + 1;
         }
         KMP_DEBUG_ASSERT( trip_count );
         switch( schedule ) {
@@ -681,8 +712,11 @@ __kmp_team_static_init(
         trip_count = upper - lower + 1;
     } else if(incr == -1) {
         trip_count = lower - upper + 1;
+    } else if ( incr > 0 ) {
+        // upper-lower can exceed the limit of signed type
+        trip_count = (UT)(upper - lower) / incr + 1;
     } else {
-        trip_count = (ST)(upper - lower) / incr + 1; // cast to signed to cover incr<0 case
+        trip_count = (UT)(lower - upper) / (-incr) + 1;
     }
     if( chunk < 1 )
         chunk = 1;

runtime/src/kmp_settings.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_settings.c -- Initialize environment variables
+ * kmp_settings.cpp -- Initialize environment variables
  */
 
 
@@ -21,7 +21,9 @@
 #include "kmp_str.h"
 #include "kmp_settings.h"
 #include "kmp_i18n.h"
+#include "kmp_lock.h"
 #include "kmp_io.h"
+#include "kmp_affinity.h"
 
 static int __kmp_env_toPrint( char const * name, int flag );
 
@@ -358,7 +360,7 @@ __kmp_stg_parse_str(
     char const *      value,
     char const * *    out
 ) {
-    KMP_INTERNAL_FREE( (void *) * out );
+    __kmp_str_free(out);
     * out = __kmp_str_format( "%s", value );
 } // __kmp_stg_parse_str
 #endif
@@ -416,12 +418,12 @@ __kmp_stg_parse_file(
     char buffer[256];
     char *t;
     int hasSuffix;
-    KMP_INTERNAL_FREE( (void *) * out );
+    __kmp_str_free(out);
     t = (char *) strrchr(value, '.');
     hasSuffix = t && __kmp_str_eqf( t, suffix );
     t = __kmp_str_format( "%s%s", value, hasSuffix ? "" : suffix );
     __kmp_expand_file_name( buffer, sizeof(buffer), t);
-    KMP_INTERNAL_FREE(t);
+    __kmp_str_free(&t);
     * out = __kmp_str_format( "%s", buffer );
 } // __kmp_stg_parse_file
 #endif
@@ -658,9 +660,11 @@ __kmp_stg_parse_blocktime( char const * name, char const * value, void * data )
         }; // if
         __kmp_env_blocktime = TRUE;    // KMP_BLOCKTIME was specified.
     }; // if
-    // calculate number of monitor thread wakeup intervals corresonding to blocktime.
+#if KMP_USE_MONITOR
+    // calculate number of monitor thread wakeup intervals corresponding to blocktime.
     __kmp_monitor_wakeups = KMP_WAKEUPS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
     __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
+#endif
     K_DIAG( 1, ( "__kmp_env_blocktime == %d\n", __kmp_env_blocktime ) );
     if ( __kmp_env_blocktime ) {
         K_DIAG( 1, ( "__kmp_dflt_blocktime == %d\n", __kmp_dflt_blocktime ) );
@@ -712,6 +716,8 @@ __kmp_stg_print_inherit_fp_control( kmp_str_buf_t * buffer, char const * name, v
 // KMP_LIBRARY, OMP_WAIT_POLICY
 // -------------------------------------------------------------------------------------------------
 
+static char const *blocktime_str = NULL;
+
 static void
 __kmp_stg_parse_wait_policy( char const * name, char const * value, void * data ) {
 
@@ -725,9 +731,17 @@ __kmp_stg_parse_wait_policy( char const * name, char const * value, void * data
 
     if ( wait->omp ) {
         if ( __kmp_str_match( "ACTIVE", 1, value ) ) {
-           __kmp_library = library_turnaround;
+            __kmp_library = library_turnaround;
+            if ( blocktime_str == NULL ) {
+                // KMP_BLOCKTIME not specified, so set default to "infinite".
+                __kmp_dflt_blocktime = KMP_MAX_BLOCKTIME;
+            }
         } else if ( __kmp_str_match( "PASSIVE", 1, value ) ) {
-           __kmp_library = library_throughput;
+            __kmp_library = library_throughput;
+            if ( blocktime_str == NULL ) {
+                // KMP_BLOCKTIME not specified, so set default to 0.
+                __kmp_dflt_blocktime = 0;
+            }
         } else {
             KMP_WARNING( StgInvalidValue, name, value );
         }; // if
@@ -784,6 +798,7 @@ __kmp_stg_print_wait_policy( kmp_str_buf_t * buffer, char const * name, void * d
 
 } // __kmp_stg_print_wait_policy
 
+#if KMP_USE_MONITOR
 // -------------------------------------------------------------------------------------------------
 // KMP_MONITOR_STACKSIZE
 // -------------------------------------------------------------------------------------------------
@@ -821,6 +836,7 @@ __kmp_stg_print_monitor_stacksize( kmp_str_buf_t * buffer, char const * name, vo
     }
 
 } // __kmp_stg_print_monitor_stacksize
+#endif // KMP_USE_MONITOR
 
 // -------------------------------------------------------------------------------------------------
 // KMP_SETTINGS
@@ -1163,7 +1179,7 @@ __kmp_stg_print_task_stealing( kmp_str_buf_t * buffer, char const * name, void *
 
 static void
 __kmp_stg_parse_max_active_levels( char const * name, char const * value, void * data ) {
-	 __kmp_stg_parse_int( name, value, 0, KMP_MAX_ACTIVE_LEVELS_LIMIT, & __kmp_dflt_max_active_levels );
+    __kmp_stg_parse_int( name, value, 0, KMP_MAX_ACTIVE_LEVELS_LIMIT, & __kmp_dflt_max_active_levels );
 } // __kmp_stg_parse_max_active_levels
 
 static void
@@ -1171,6 +1187,51 @@ __kmp_stg_print_max_active_levels( kmp_str_buf_t * buffer, char const * name, vo
     __kmp_stg_print_int( buffer, name, __kmp_dflt_max_active_levels );
 } // __kmp_stg_print_max_active_levels
 
+#if OMP_40_ENABLED
+// -------------------------------------------------------------------------------------------------
+// OpenMP 4.0: OMP_DEFAULT_DEVICE
+// -------------------------------------------------------------------------------------------------
+static void __kmp_stg_parse_default_device(char const *name, char const *value, void *data) {
+  __kmp_stg_parse_int(name, value, 0, KMP_MAX_DEFAULT_DEVICE_LIMIT, &__kmp_default_device);
+} // __kmp_stg_parse_default_device
+
+static void __kmp_stg_print_default_device(kmp_str_buf_t *buffer, char const *name, void *data) {
+  __kmp_stg_print_int(buffer, name, __kmp_default_device);
+} // __kmp_stg_print_default_device
+#endif
+
+#if OMP_45_ENABLED
+// -------------------------------------------------------------------------------------------------
+// OpenMP 4.5: OMP_MAX_TASK_PRIORITY
+// -------------------------------------------------------------------------------------------------
+static void
+__kmp_stg_parse_max_task_priority(char const *name, char const *value, void *data) {
+    __kmp_stg_parse_int(name, value, 0, KMP_MAX_TASK_PRIORITY_LIMIT, &__kmp_max_task_priority);
+} // __kmp_stg_parse_max_task_priority
+
+static void
+__kmp_stg_print_max_task_priority(kmp_str_buf_t *buffer, char const *name, void *data) {
+    __kmp_stg_print_int(buffer, name, __kmp_max_task_priority);
+} // __kmp_stg_print_max_task_priority
+#endif // OMP_45_ENABLED
+
+// -------------------------------------------------------------------------------------------------
+// KMP_DISP_NUM_BUFFERS
+// -------------------------------------------------------------------------------------------------
+static void
+__kmp_stg_parse_disp_buffers( char const * name, char const * value, void * data ) {
+    if ( TCR_4(__kmp_init_serial) ) {
+        KMP_WARNING( EnvSerialWarn, name );
+        return;
+    }   // read value before serial initialization only
+    __kmp_stg_parse_int( name, value, 1, KMP_MAX_NTH, & __kmp_dispatch_num_buffers );
+} // __kmp_stg_parse_disp_buffers
+
+static void
+__kmp_stg_print_disp_buffers( kmp_str_buf_t * buffer, char const * name, void * data ) {
+    __kmp_stg_print_int( buffer, name, __kmp_dispatch_num_buffers );
+} // __kmp_stg_print_disp_buffers
+
 #if KMP_NESTED_HOT_TEAMS
 // -------------------------------------------------------------------------------------------------
 // KMP_HOT_TEAMS_MAX_LEVEL, KMP_HOT_TEAMS_MODE
@@ -2174,7 +2235,7 @@ __kmp_parse_affinity_env( char const * name, char const * value,
     #undef set_respect
     #undef set_granularity
 
-    KMP_INTERNAL_FREE( buffer );
+    __kmp_str_free((const char **) &buffer);
 
     if ( proclist ) {
         if ( ! type ) {
@@ -3500,9 +3561,8 @@ __kmp_stg_parse_omp_schedule( char const * name, char const * value, void * data
                 __kmp_sched = kmp_sch_trapezoidal;
             else if (!__kmp_strcasecmp_with_sentinel("static", value, ','))      /* STATIC */
                 __kmp_sched = kmp_sch_static;
-#ifdef KMP_STATIC_STEAL_ENABLED
-            else if (KMP_ARCH_X86_64 &&
-                     !__kmp_strcasecmp_with_sentinel("static_steal", value, ','))
+#if KMP_STATIC_STEAL_ENABLED
+            else if (!__kmp_strcasecmp_with_sentinel("static_steal", value, ','))
                 __kmp_sched = kmp_sch_static_steal;
 #endif
             else {
@@ -3738,6 +3798,7 @@ __kmp_stg_print_par_range_env( kmp_str_buf_t * buffer, char const * name, void *
     }
 } // __kmp_stg_print_par_range_env
 
+#if KMP_USE_MONITOR
 // -------------------------------------------------------------------------------------------------
 // KMP_YIELD_CYCLE, KMP_YIELD_ON, KMP_YIELD_OFF
 // -------------------------------------------------------------------------------------------------
@@ -3773,6 +3834,7 @@ static void
 __kmp_stg_print_yield_off( kmp_str_buf_t * buffer, char const * name, void * data ) {
     __kmp_stg_print_int( buffer, name, __kmp_yield_off_count );
 } // __kmp_stg_print_yield_off
+#endif // KMP_USE_MONITOR
 
 #endif
 
@@ -3854,7 +3916,6 @@ __kmp_stg_print_gtid_mode( kmp_str_buf_t * buffer, char const * name, void * dat
     }
 } // __kmp_stg_print_gtid_mode
 
-
 // -------------------------------------------------------------------------------------------------
 // KMP_NUM_LOCKS_IN_BLOCK
 // -------------------------------------------------------------------------------------------------
@@ -3900,7 +3961,7 @@ __kmp_stg_parse_lock_kind( char const * name, char const * value, void * data )
         __kmp_user_lock_kind = lk_tas;
         KMP_STORE_LOCK_SEQ(tas);
     }
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
+#if KMP_USE_FUTEX
     else if ( __kmp_str_match( "futex", 1, value ) ) {
         if ( __kmp_futex_determine_capable() ) {
             __kmp_user_lock_kind = lk_futex;
@@ -3974,7 +4035,7 @@ __kmp_stg_print_lock_kind( kmp_str_buf_t * buffer, char const * name, void * dat
         value = "tas";
         break;
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
+#if KMP_USE_FUTEX
         case lk_futex:
         value = "futex";
         break;
@@ -4013,6 +4074,102 @@ __kmp_stg_print_lock_kind( kmp_str_buf_t * buffer, char const * name, void * dat
     }
 }
 
+// -------------------------------------------------------------------------------------------------
+// KMP_SPIN_BACKOFF_PARAMS
+// -------------------------------------------------------------------------------------------------
+
+// KMP_SPIN_BACKOFF_PARAMS=max_backoff[,min_tick] (max backoff size, min tick for machine pause)
+static void
+__kmp_stg_parse_spin_backoff_params(const char* name, const char* value, void* data)
+{
+    const char *next = value;
+
+    int total = 0;          // Count elements that were set. It'll be used as an array size
+    int prev_comma = FALSE; // For correct processing sequential commas
+    int i;
+
+    kmp_uint32 max_backoff = __kmp_spin_backoff_params.max_backoff;
+    kmp_uint32 min_tick = __kmp_spin_backoff_params.min_tick;
+
+    // Run only 3 iterations because it is enough to read two values or find a syntax error
+    for ( i = 0; i < 3 ; i++) {
+        SKIP_WS( next );
+
+        if ( *next == '\0' ) {
+            break;
+        }
+        // Next character is not an integer or not a comma OR number of values > 2 => end of list
+        if ( ( ( *next < '0' || *next > '9' ) && *next !=',' ) || total > 2 ) {
+            KMP_WARNING( EnvSyntaxError, name, value );
+            return;
+        }
+        // The next character is ','
+        if ( *next == ',' ) {
+            // ',' is the fisrt character
+            if ( total == 0 || prev_comma ) {
+                total++;
+            }
+            prev_comma = TRUE;
+            next++; //skip ','
+            SKIP_WS( next );
+        }
+        // Next character is a digit
+        if ( *next >= '0' && *next <= '9' ) {
+            int num;
+            const char *buf = next;
+            char const * msg  = NULL;
+            prev_comma = FALSE;
+            SKIP_DIGITS( next );
+            total++;
+
+            const char *tmp = next;
+            SKIP_WS( tmp );
+            if ( ( *next == ' ' || *next == '\t' ) && ( *tmp >= '0' && *tmp <= '9' ) ) {
+                KMP_WARNING( EnvSpacesNotAllowed, name, value );
+                return;
+            }
+
+            num = __kmp_str_to_int( buf, *next );
+            if ( num <= 0 ) { // The number of retries should be > 0
+                msg = KMP_I18N_STR( ValueTooSmall );
+                num = 1;
+            } else if ( num > KMP_INT_MAX ) {
+                msg = KMP_I18N_STR( ValueTooLarge );
+                num = KMP_INT_MAX;
+            }
+            if ( msg != NULL ) {
+                // Message is not empty. Print warning.
+                KMP_WARNING( ParseSizeIntWarn, name, value, msg );
+                KMP_INFORM( Using_int_Value, name, num );
+            }
+            if( total == 1 ) {
+                max_backoff = num;
+            } else if( total == 2 ) {
+                min_tick = num;
+            }
+        }
+    }
+    KMP_DEBUG_ASSERT( total > 0 );
+    if( total <= 0 ) {
+        KMP_WARNING( EnvSyntaxError, name, value );
+        return;
+    }
+    __kmp_spin_backoff_params.max_backoff = max_backoff;
+    __kmp_spin_backoff_params.min_tick    = min_tick;
+}
+
+static void
+__kmp_stg_print_spin_backoff_params(kmp_str_buf_t *buffer, char const* name, void* data)
+{
+    if( __kmp_env_format ) {
+        KMP_STR_BUF_PRINT_NAME_EX(name);
+    } else {
+        __kmp_str_buf_print( buffer, "   %s='", name );
+    }
+    __kmp_str_buf_print( buffer, "%d,%d'\n", __kmp_spin_backoff_params.max_backoff,
+                         __kmp_spin_backoff_params.min_tick );
+}
+
 #if KMP_USE_ADAPTIVE_LOCKS
 
 // -------------------------------------------------------------------------------------------------
@@ -4136,11 +4293,11 @@ __kmp_stg_print_speculative_statsfile( kmp_str_buf_t * buffer, char const * name
 #endif // KMP_USE_ADAPTIVE_LOCKS
 
 // -------------------------------------------------------------------------------------------------
-// KMP_PLACE_THREADS
+// KMP_HW_SUBSET (was KMP_PLACE_THREADS)
 // -------------------------------------------------------------------------------------------------
 
 static void
-__kmp_stg_parse_place_threads( char const * name, char const * value, void * data ) {
+__kmp_stg_parse_hw_subset( char const * name, char const * value, void * data ) {
     // Value example: 5Cx2Tx15O
     // Which means "use 5 cores with offset 15, 2 threads per core"
     // AC: extended to sockets level, examples of
@@ -4150,12 +4307,32 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
     //     postfix "o" or prefix @ can be offset designator.
     // Note: not all syntax errors are analyzed, some may be skipped.
 #define CHECK_DELIM(_x)   (*(_x) == ',' || *(_x) == 'x')
+    static int parsed = 0;
     int         num;
     int single_warning = 0;
     int flagS = 0, flagC = 0, flagT = 0, flagSO = 0, flagCO = 0;
     const char *next = value;
     const char *prev;
 
+    if( strcmp(name, "KMP_PLACE_THREADS") == 0 ) {
+        KMP_INFORM(EnvVarDeprecated,name,"KMP_HW_SUBSET");
+        if( parsed == 1 ) {
+            return; // already parsed KMP_HW_SUBSET
+        }
+    }
+    parsed = 1;
+
+    SKIP_WS(next);  // skip white spaces
+    if (*next == '\0')
+        return;   // no data provided, retain default values
+    if( strcmp(name, "KMP_PLACE_THREADS") == 0 ) {
+        KMP_INFORM(EnvVarDeprecated,name,"KMP_HW_SUBSET");
+        if( parsed == 1 ) {
+            return; // already parsed KMP_HW_SUBSET
+        }
+    }
+    parsed = 1;
+
     SKIP_WS(next);  // skip white spaces
     if (*next == '\0')
         return;   // no data provided, retain default values
@@ -4173,7 +4350,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
                 flagSO = 1;
                 prev = ++next;  // don't allow spaces for simplicity
                 if (!(*next >= '0' && *next <= '9')) {
-                    KMP_WARNING(AffThrPlaceInvalid, name, value);
+                    KMP_WARNING(AffHWSubsetInvalid, name, value);
                     return;
                 }
                 SKIP_DIGITS(next);
@@ -4188,7 +4365,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
                 flagCO = 1;
                 prev = ++next;  // don't allow spaces for simplicity
                 if (!(*next >= '0' && *next <= '9')) {
-                    KMP_WARNING(AffThrPlaceInvalid, name, value);
+                    KMP_WARNING(AffHWSubsetInvalid, name, value);
                     return;
                 }
                 SKIP_DIGITS(next);
@@ -4207,11 +4384,11 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
             __kmp_place_num_cores = num;
             return;   // the only value provided - set num cores
         } else {
-            KMP_WARNING(AffThrPlaceInvalid, name, value);
+            KMP_WARNING(AffHWSubsetInvalid, name, value);
             return;
         }
     } else {
-        KMP_WARNING(AffThrPlaceInvalid, name, value);
+        KMP_WARNING(AffHWSubsetInvalid, name, value);
         return;
     }
     KMP_DEBUG_ASSERT(flagS); // num sockets should already be set here
@@ -4238,7 +4415,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
                 flagCO = 1;
                 prev = ++next;  // don't allow spaces for simplicity
                 if (!(*next >= '0' && *next <= '9')) {
-                    KMP_WARNING(AffThrPlaceInvalid, name, value);
+                    KMP_WARNING(AffHWSubsetInvalid, name, value);
                     return;
                 }
                 SKIP_DIGITS(next);
@@ -4246,7 +4423,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
                 __kmp_place_core_offset = num;
             }
         } else if (*next == 'o' || *next == 'O') { // offset specified
-            KMP_WARNING(AffThrPlaceDeprecated);
+            KMP_WARNING(AffHWSubsetDeprecated);
             single_warning = 1;
             if (flagC) { // whether num_cores already specified (sockets skipped)
                 KMP_DEBUG_ASSERT(!flagCO); // either "o" or @, not both
@@ -4267,11 +4444,11 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
             __kmp_place_num_threads_per_core = num;
             return;   // two values provided without letter-designator
         } else {
-            KMP_WARNING(AffThrPlaceInvalid, name, value);
+            KMP_WARNING(AffHWSubsetInvalid, name, value);
             return;
         }
     } else {
-        KMP_WARNING(AffThrPlaceInvalid, name, value);
+        KMP_WARNING(AffHWSubsetInvalid, name, value);
         return;
     }
     SKIP_WS(next);
@@ -4302,18 +4479,18 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
             next++;
             //KMP_DEBUG_ASSERT(*next != '@'); // socket offset used "o" designator
         } else if (*next == 'o' || *next == 'O') {
-            KMP_WARNING(AffThrPlaceDeprecated);
+            KMP_WARNING(AffHWSubsetDeprecated);
             single_warning = 1;
             KMP_DEBUG_ASSERT(flagC);
             //KMP_DEBUG_ASSERT(!flagSO); // socket offset couldn't use @ designator
             __kmp_place_core_offset = num;
             next++;
         } else {
-            KMP_WARNING(AffThrPlaceInvalid, name, value);
+            KMP_WARNING(AffHWSubsetInvalid, name, value);
             return;
         }
     } else {
-        KMP_WARNING(AffThrPlaceInvalid, name, value);
+        KMP_WARNING(AffHWSubsetInvalid, name, value);
         return;
     }
     KMP_DEBUG_ASSERT(flagC);
@@ -4333,7 +4510,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
         SKIP_WS(next);
         if (*next == 'o' || *next == 'O') {
             if (!single_warning) { // warn once
-                KMP_WARNING(AffThrPlaceDeprecated);
+                KMP_WARNING(AffHWSubsetDeprecated);
             }
             KMP_DEBUG_ASSERT(!flagSO); // socket offset couldn't use @ designator
             __kmp_place_core_offset = num;
@@ -4344,11 +4521,11 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
             flagT = 1;
             next++; // can have core-offset specified after num threads
         } else {
-            KMP_WARNING(AffThrPlaceInvalid, name, value);
+            KMP_WARNING(AffHWSubsetInvalid, name, value);
             return;
         }
     } else {
-        KMP_WARNING(AffThrPlaceInvalid, name, value);
+        KMP_WARNING(AffHWSubsetInvalid, name, value);
         return;
     }
     SKIP_WS(next);
@@ -4367,7 +4544,7 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
         SKIP_WS(next);
         if (*next == 'o' || *next == 'O') {
             if (!single_warning) { // warn once
-                KMP_WARNING(AffThrPlaceDeprecated);
+                KMP_WARNING(AffHWSubsetDeprecated);
             }
             KMP_DEBUG_ASSERT(flagT);
             KMP_DEBUG_ASSERT(!flagSO); // socket offset couldn't use @ designator
@@ -4376,17 +4553,17 @@ __kmp_stg_parse_place_threads( char const * name, char const * value, void * dat
             KMP_DEBUG_ASSERT(flagT == 0);
             __kmp_place_num_threads_per_core = num;
         } else {
-            KMP_WARNING(AffThrPlaceInvalid, name, value);
+            KMP_WARNING(AffHWSubsetInvalid, name, value);
         }
     } else {
-        KMP_WARNING(AffThrPlaceInvalid, name, value);
+        KMP_WARNING(AffHWSubsetInvalid, name, value);
     }
     return;
 #undef CHECK_DELIM
 }
 
 static void
-__kmp_stg_print_place_threads( kmp_str_buf_t * buffer, char const * name, void * data ) {
+__kmp_stg_print_hw_subset( kmp_str_buf_t * buffer, char const * name, void * data ) {
     if (__kmp_place_num_sockets + __kmp_place_num_cores + __kmp_place_num_threads_per_core) {
         int comma = 0;
         kmp_str_buf_t buf;
@@ -4505,7 +4682,9 @@ static kmp_setting_t __kmp_stg_table[] = {
     { "KMP_DUPLICATE_LIB_OK",              __kmp_stg_parse_duplicate_lib_ok,   __kmp_stg_print_duplicate_lib_ok,   NULL, 0, 0 },
     { "KMP_LIBRARY",                       __kmp_stg_parse_wait_policy,        __kmp_stg_print_wait_policy,        NULL, 0, 0 },
     { "KMP_MAX_THREADS",                   __kmp_stg_parse_all_threads,        NULL,                               NULL, 0, 0 }, // For backward compatibility
+#if KMP_USE_MONITOR
     { "KMP_MONITOR_STACKSIZE",             __kmp_stg_parse_monitor_stacksize,  __kmp_stg_print_monitor_stacksize,  NULL, 0, 0 },
+#endif
     { "KMP_SETTINGS",                      __kmp_stg_parse_settings,           __kmp_stg_print_settings,           NULL, 0, 0 },
     { "KMP_STACKOFFSET",                   __kmp_stg_parse_stackoffset,        __kmp_stg_print_stackoffset,        NULL, 0, 0 },
     { "KMP_STACKSIZE",                     __kmp_stg_parse_stacksize,          __kmp_stg_print_stacksize,          NULL, 0, 0 },
@@ -4520,8 +4699,15 @@ static kmp_setting_t __kmp_stg_table[] = {
     { "KMP_TASKING",                       __kmp_stg_parse_tasking,            __kmp_stg_print_tasking,            NULL, 0, 0 },
     { "KMP_TASK_STEALING_CONSTRAINT",      __kmp_stg_parse_task_stealing,      __kmp_stg_print_task_stealing,      NULL, 0, 0 },
     { "OMP_MAX_ACTIVE_LEVELS",             __kmp_stg_parse_max_active_levels,  __kmp_stg_print_max_active_levels,  NULL, 0, 0 },
+#if OMP_40_ENABLED
+    { "OMP_DEFAULT_DEVICE",                __kmp_stg_parse_default_device,     __kmp_stg_print_default_device,     NULL, 0, 0 },
+#endif
+#if OMP_45_ENABLED
+    { "OMP_MAX_TASK_PRIORITY",             __kmp_stg_parse_max_task_priority,  __kmp_stg_print_max_task_priority,  NULL, 0, 0 },
+#endif
     { "OMP_THREAD_LIMIT",                  __kmp_stg_parse_all_threads,        __kmp_stg_print_all_threads,        NULL, 0, 0 },
     { "OMP_WAIT_POLICY",                   __kmp_stg_parse_wait_policy,        __kmp_stg_print_wait_policy,        NULL, 0, 0 },
+    { "KMP_DISP_NUM_BUFFERS",              __kmp_stg_parse_disp_buffers,       __kmp_stg_print_disp_buffers,       NULL, 0, 0 },
 #if KMP_NESTED_HOT_TEAMS
     { "KMP_HOT_TEAMS_MAX_LEVEL",           __kmp_stg_parse_hot_teams_level,    __kmp_stg_print_hot_teams_level,    NULL, 0, 0 },
     { "KMP_HOT_TEAMS_MODE",                __kmp_stg_parse_hot_teams_mode,     __kmp_stg_print_hot_teams_mode,     NULL, 0, 0 },
@@ -4554,9 +4740,11 @@ static kmp_setting_t __kmp_stg_table[] = {
     { "KMP_DIAG",                          __kmp_stg_parse_diag,               __kmp_stg_print_diag,               NULL, 0, 0 },
 
     { "KMP_PAR_RANGE",                     __kmp_stg_parse_par_range_env,      __kmp_stg_print_par_range_env,      NULL, 0, 0 },
+#if KMP_USE_MONITOR
     { "KMP_YIELD_CYCLE",                   __kmp_stg_parse_yield_cycle,        __kmp_stg_print_yield_cycle,        NULL, 0, 0 },
     { "KMP_YIELD_ON",                      __kmp_stg_parse_yield_on,           __kmp_stg_print_yield_on,           NULL, 0, 0 },
     { "KMP_YIELD_OFF",                     __kmp_stg_parse_yield_off,          __kmp_stg_print_yield_off,          NULL, 0, 0 },
+#endif
 #endif // KMP_DEBUG
 
     { "KMP_ALIGN_ALLOC",                   __kmp_stg_parse_align_alloc,        __kmp_stg_print_align_alloc,        NULL, 0, 0 },
@@ -4626,13 +4814,15 @@ static kmp_setting_t __kmp_stg_table[] = {
 
     { "KMP_NUM_LOCKS_IN_BLOCK",            __kmp_stg_parse_lock_block,         __kmp_stg_print_lock_block,         NULL, 0, 0 },
     { "KMP_LOCK_KIND",                     __kmp_stg_parse_lock_kind,          __kmp_stg_print_lock_kind,          NULL, 0, 0 },
+    { "KMP_SPIN_BACKOFF_PARAMS",           __kmp_stg_parse_spin_backoff_params, __kmp_stg_print_spin_backoff_params, NULL, 0, 0 },
 #if KMP_USE_ADAPTIVE_LOCKS
     { "KMP_ADAPTIVE_LOCK_PROPS",           __kmp_stg_parse_adaptive_lock_props,__kmp_stg_print_adaptive_lock_props,  NULL, 0, 0 },
 #if KMP_DEBUG_ADAPTIVE_LOCKS
     { "KMP_SPECULATIVE_STATSFILE",         __kmp_stg_parse_speculative_statsfile,__kmp_stg_print_speculative_statsfile,  NULL, 0, 0 },
 #endif
 #endif // KMP_USE_ADAPTIVE_LOCKS
-    { "KMP_PLACE_THREADS",                 __kmp_stg_parse_place_threads,      __kmp_stg_print_place_threads,      NULL, 0, 0 },
+    { "KMP_PLACE_THREADS",                 __kmp_stg_parse_hw_subset,      __kmp_stg_print_hw_subset,      NULL, 0, 0 },
+    { "KMP_HW_SUBSET",                     __kmp_stg_parse_hw_subset,      __kmp_stg_print_hw_subset,      NULL, 0, 0 },
 #if USE_ITT_BUILD
     { "KMP_FORKJOIN_FRAMES",               __kmp_stg_parse_forkjoin_frames,    __kmp_stg_print_forkjoin_frames,    NULL, 0, 0 },
     { "KMP_FORKJOIN_FRAMES_MODE",          __kmp_stg_parse_forkjoin_frames_mode,__kmp_stg_print_forkjoin_frames_mode,  NULL, 0, 0 },
@@ -5008,6 +5198,9 @@ __kmp_env_initialize( char const * string ) {
         }
     }; // for i
 
+    // We need to know if blocktime was set when processing OMP_WAIT_POLICY
+    blocktime_str = __kmp_env_blk_var( & block, "KMP_BLOCKTIME" );
+
     // Special case. If we parse environment, not a string, process KMP_WARNINGS first.
     if ( string == NULL ) {
         char const * name  = "KMP_WARNINGS";
@@ -5147,43 +5340,12 @@ __kmp_env_initialize( char const * string ) {
         // affinity.
         //
         const char *var = "KMP_AFFINITY";
-# if KMP_USE_HWLOC
-        if(hwloc_topology_init(&__kmp_hwloc_topology) < 0) {
-            __kmp_hwloc_error = TRUE;
-            if(__kmp_affinity_verbose)
-                KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_init()");
-        }
-        hwloc_topology_ignore_type(__kmp_hwloc_topology, HWLOC_OBJ_CACHE);
-# endif
+        KMPAffinity::pick_api();
         if ( __kmp_affinity_type == affinity_disabled ) {
             KMP_AFFINITY_DISABLE();
         }
         else if ( ! KMP_AFFINITY_CAPABLE() ) {
-# if KMP_USE_HWLOC
-            const hwloc_topology_support* topology_support = hwloc_topology_get_support(__kmp_hwloc_topology);
-            if(hwloc_topology_load(__kmp_hwloc_topology) < 0) {
-                __kmp_hwloc_error = TRUE;
-                if(__kmp_affinity_verbose)
-                    KMP_WARNING(AffHwlocErrorOccurred, var, "hwloc_topology_load()");
-            }
-            // Is the system capable of setting/getting this thread's affinity?
-            // also, is topology discovery possible? (pu indicates ability to discover processing units)
-            // and finally, were there no errors when calling any hwloc_* API functions?
-            if(topology_support->cpubind->set_thisthread_cpubind &&
-               topology_support->cpubind->get_thisthread_cpubind &&
-               topology_support->discovery->pu &&
-               !__kmp_hwloc_error)
-            {
-                // enables affinity according to KMP_AFFINITY_CAPABLE() macro
-                KMP_AFFINITY_ENABLE(TRUE);
-            } else {
-                // indicate that hwloc didn't work and disable affinity
-                __kmp_hwloc_error = TRUE;
-                KMP_AFFINITY_DISABLE();
-            }
-# else
-            __kmp_affinity_determine_capable( var );
-# endif // KMP_USE_HWLOC
+            __kmp_affinity_dispatch->determine_capable(var);
             if ( ! KMP_AFFINITY_CAPABLE() ) {
                 if ( __kmp_affinity_verbose || ( __kmp_affinity_warnings
                   && ( __kmp_affinity_type != affinity_default )
@@ -5252,7 +5414,7 @@ __kmp_env_initialize( char const * string ) {
                     else if ( ( __kmp_affinity_gran != affinity_gran_group )
                       && ( __kmp_affinity_gran != affinity_gran_fine )
                       && ( __kmp_affinity_gran != affinity_gran_thread ) ) {
-                        char *str = NULL;
+                        const char *str = NULL;
                         switch ( __kmp_affinity_gran ) {
                             case affinity_gran_core: str = "core"; break;
                             case affinity_gran_package: str = "package"; break;
@@ -5268,7 +5430,7 @@ __kmp_env_initialize( char const * string ) {
                         __kmp_affinity_gran = affinity_gran_core;
                     }
                     else if ( __kmp_affinity_gran == affinity_gran_group ) {
-                        char *str = NULL;
+                        const char *str = NULL;
                         switch ( __kmp_affinity_type ) {
                             case affinity_physical: str = "physical"; break;
                             case affinity_logical: str = "logical"; break;

runtime/src/kmp_stats.cpp

@@ -21,6 +21,7 @@
 #include <sstream>
 #include <iomanip>
 #include <stdlib.h>                             // for atexit
+#include <ctime>
 
 #define STRINGIZE2(x) #x
 #define STRINGIZE(x) STRINGIZE2(x)
@@ -28,11 +29,11 @@
 #define expandName(name,flags,ignore)  {STRINGIZE(name),flags},
 statInfo timeStat::timerInfo[] = {
     KMP_FOREACH_TIMER(expandName,0)
-    {0,0}
+    {"TIMER_LAST", 0}
 };
 const statInfo counter::counterInfo[] = {
     KMP_FOREACH_COUNTER(expandName,0)
-    {0,0}
+    {"COUNTER_LAST", 0}
 };
 #undef expandName
 
@@ -47,7 +48,7 @@ const kmp_stats_output_module::rgb_color kmp_stats_output_module::globalColorArr
     {1.0, 0.0, 0.0}, // red
     {1.0, 0.6, 0.0}, // orange
     {1.0, 1.0, 0.0}, // yellow
-    {0.0, 1.0, 0.0}, // green 
+    {0.0, 1.0, 0.0}, // green
     {0.0, 0.0, 1.0}, // blue
     {0.6, 0.2, 0.8}, // purple
     {1.0, 0.0, 1.0}, // magenta
@@ -70,7 +71,7 @@ const kmp_stats_output_module::rgb_color kmp_stats_output_module::globalColorArr
 static uint32_t statsPrinted = 0;
 
 // output interface
-static kmp_stats_output_module __kmp_stats_global_output;
+static kmp_stats_output_module* __kmp_stats_global_output = NULL;
 
 /* ****************************************************** */
 /* ************* statistic member functions ************* */
@@ -130,46 +131,128 @@ std::string statistic::format(char unit, bool total) const
 {
     std::string result = formatSI(sampleCount,9,' ');
 
-    result = result + std::string(", ") + formatSI(minVal,  9, unit);
-    result = result + std::string(", ") + formatSI(meanVal, 9, unit);
-    result = result + std::string(", ") + formatSI(maxVal,  9, unit);
-    if (total)
-        result = result + std::string(", ") + formatSI(meanVal*sampleCount, 9, unit);
-    result = result + std::string(", ") + formatSI(getSD(), 9, unit);
-
+    if (sampleCount == 0)
+    {
+        result = result + std::string(", ") + formatSI(0.0, 9, unit);
+        result = result + std::string(", ") + formatSI(0.0, 9, unit);
+        result = result + std::string(", ") + formatSI(0.0, 9, unit);
+        if (total)
+            result = result + std::string(", ") + formatSI(0.0, 9, unit);
+        result = result + std::string(", ") + formatSI(0.0, 9, unit);
+    }
+    else
+    {
+        result = result + std::string(", ") + formatSI(minVal,  9, unit);
+        result = result + std::string(", ") + formatSI(meanVal, 9, unit);
+        result = result + std::string(", ") + formatSI(maxVal,  9, unit);
+        if (total)
+            result = result + std::string(", ") + formatSI(meanVal*sampleCount, 9, unit);
+        result = result + std::string(", ") + formatSI(getSD(), 9, unit);
+    }
     return result;
 }
 
 /* ********************************************************** */
 /* ************* explicitTimer member functions ************* */
 
-void explicitTimer::start(timer_e timerEnumValue) { 
-    startTime = tsc_tick_count::now(); 
+void explicitTimer::start(timer_e timerEnumValue) {
+    startTime = tsc_tick_count::now();
+    totalPauseTime = 0;
     if(timeStat::logEvent(timerEnumValue)) {
         __kmp_stats_thread_ptr->incrementNestValue();
     }
     return;
 }
 
-void explicitTimer::stop(timer_e timerEnumValue) {
+void explicitTimer::stop(timer_e timerEnumValue, kmp_stats_list* stats_ptr /* = nullptr */) {
     if (startTime.getValue() == 0)
         return;
 
     tsc_tick_count finishTime = tsc_tick_count::now();
 
     //stat->addSample ((tsc_tick_count::now() - startTime).ticks());
-    stat->addSample ((finishTime - startTime).ticks());
+    stat->addSample(((finishTime - startTime) - totalPauseTime).ticks());
 
     if(timeStat::logEvent(timerEnumValue)) {
-        __kmp_stats_thread_ptr->push_event(startTime.getValue() - __kmp_stats_start_time.getValue(), finishTime.getValue() - __kmp_stats_start_time.getValue(), __kmp_stats_thread_ptr->getNestValue(), timerEnumValue); 
-        __kmp_stats_thread_ptr->decrementNestValue();
+        if(!stats_ptr)
+            stats_ptr = __kmp_stats_thread_ptr;
+        stats_ptr->push_event(startTime.getValue() - __kmp_stats_start_time.getValue(), finishTime.getValue() - __kmp_stats_start_time.getValue(), __kmp_stats_thread_ptr->getNestValue(), timerEnumValue);
+        stats_ptr->decrementNestValue();
     }
 
     /* We accept the risk that we drop a sample because it really did start at t==0. */
-    startTime = 0; 
+    startTime = 0;
     return;
 }
 
+/* ************************************************************** */
+/* ************* partitionedTimers member functions ************* */
+partitionedTimers::partitionedTimers() {
+    timer_stack.reserve(8);
+}
+
+// add a timer to this collection of partitioned timers.
+void partitionedTimers::add_timer(explicit_timer_e timer_index, explicitTimer* timer_pointer) {
+    KMP_DEBUG_ASSERT((int)timer_index < (int)EXPLICIT_TIMER_LAST+1);
+    timers[timer_index] = timer_pointer;
+}
+
+// initialize the paritioned timers to an initial timer
+void partitionedTimers::init(timerPair init_timer_pair) {
+    KMP_DEBUG_ASSERT(this->timer_stack.size() == 0);
+    timer_stack.push_back(init_timer_pair);
+    timers[init_timer_pair.get_index()]->start(init_timer_pair.get_timer());
+}
+
+// stop/save the current timer, and start the new timer (timer_pair)
+// There is a special condition where if the current timer is equal to
+// the one you are trying to push, then it only manipulates the stack,
+// and it won't stop/start the currently running timer.
+void partitionedTimers::push(timerPair timer_pair) {
+    // get the current timer
+    // stop current timer
+    // push new timer
+    // start the new timer
+    KMP_DEBUG_ASSERT(this->timer_stack.size() > 0);
+    timerPair current_timer = timer_stack.back();
+    timer_stack.push_back(timer_pair);
+    if(current_timer != timer_pair) {
+        timers[current_timer.get_index()]->pause();
+        timers[timer_pair.get_index()]->start(timer_pair.get_timer());
+    }
+}
+
+// stop/discard the current timer, and start the previously saved timer
+void partitionedTimers::pop() {
+    // get the current timer
+    // stop current timer
+    // pop current timer
+    // get the new current timer and start it back up
+    KMP_DEBUG_ASSERT(this->timer_stack.size() > 1);
+    timerPair current_timer = timer_stack.back();
+    timer_stack.pop_back();
+    timerPair new_timer = timer_stack.back();
+    if(current_timer != new_timer) {
+        timers[current_timer.get_index()]->stop(current_timer.get_timer());
+        timers[new_timer.get_index()]->resume();
+    }
+}
+
+// Wind up all the currently running timers.
+// This pops off all the timers from the stack and clears the stack
+// After this is called, init() must be run again to initialize the
+// stack of timers
+void partitionedTimers::windup() {
+    while(timer_stack.size() > 1) {
+        this->pop();
+    }
+    if(timer_stack.size() > 0) {
+        timerPair last_timer = timer_stack.back();
+        timer_stack.pop_back();
+        timers[last_timer.get_index()]->stop(last_timer.get_timer());
+    }
+}
+
 /* ******************************************************************* */
 /* ************* kmp_stats_event_vector member functions ************* */
 
@@ -182,7 +265,7 @@ void kmp_stats_event_vector::deallocate() {
 
 // This function is for qsort() which requires the compare function to return
 // either a negative number if event1 < event2, a positive number if event1 > event2
-// or zero if event1 == event2.  
+// or zero if event1 == event2.
 // This sorts by start time (lowest to highest).
 int compare_two_events(const void* event1, const void* event2) {
     kmp_stats_event* ev1 = (kmp_stats_event*)event1;
@@ -201,7 +284,7 @@ void kmp_stats_event_vector::sort() {
 /* ************* kmp_stats_list member functions ************* */
 
 // returns a pointer to newly created stats node
-kmp_stats_list* kmp_stats_list::push_back(int gtid) { 
+kmp_stats_list* kmp_stats_list::push_back(int gtid) {
     kmp_stats_list* newnode = (kmp_stats_list*)__kmp_allocate(sizeof(kmp_stats_list));
     // placement new, only requires space and pointer and initializes (so __kmp_allocate instead of C++ new[] is used)
     new (newnode) kmp_stats_list();
@@ -244,7 +327,7 @@ int kmp_stats_list::size() {
 /* ********************************************************************* */
 /* ************* kmp_stats_list::iterator member functions ************* */
 
-kmp_stats_list::iterator::iterator() : ptr(NULL) {} 
+kmp_stats_list::iterator::iterator() : ptr(NULL) {}
 kmp_stats_list::iterator::~iterator() {}
 kmp_stats_list::iterator kmp_stats_list::iterator::operator++() {
     this->ptr = this->ptr->next;
@@ -263,10 +346,10 @@ kmp_stats_list::iterator kmp_stats_list::iterator::operator--(int dummy) {
     return *this;
 }
 bool kmp_stats_list::iterator::operator!=(const kmp_stats_list::iterator & rhs) {
-   return this->ptr!=rhs.ptr; 
+   return this->ptr!=rhs.ptr;
 }
 bool kmp_stats_list::iterator::operator==(const kmp_stats_list::iterator & rhs) {
-   return this->ptr==rhs.ptr; 
+   return this->ptr==rhs.ptr;
 }
 kmp_stats_list* kmp_stats_list::iterator::operator*() const {
     return this->ptr;
@@ -275,14 +358,13 @@ kmp_stats_list* kmp_stats_list::iterator::operator*() const {
 /* *************************************************************** */
 /* *************  kmp_stats_output_module functions ************** */
 
-const char* kmp_stats_output_module::outputFileName = NULL;
 const char* kmp_stats_output_module::eventsFileName = NULL;
 const char* kmp_stats_output_module::plotFileName   = NULL;
 int kmp_stats_output_module::printPerThreadFlag       = 0;
 int kmp_stats_output_module::printPerThreadEventsFlag = 0;
 
 // init() is called very near the beginning of execution time in the constructor of __kmp_stats_global_output
-void kmp_stats_output_module::init() 
+void kmp_stats_output_module::init()
 {
     char * statsFileName  = getenv("KMP_STATS_FILE");
     eventsFileName        = getenv("KMP_STATS_EVENTS_FILE");
@@ -291,7 +373,24 @@ void kmp_stats_output_module::init()
     char * threadEvents   = getenv("KMP_STATS_EVENTS");
 
     // set the stats output filenames based on environment variables and defaults
-    outputFileName = statsFileName;
+    if(statsFileName) {
+        // append the process id to the output filename
+        // events.csv --> events-pid.csv
+        size_t index;
+        std::string baseFileName, pid, suffix;
+        std::stringstream ss;
+        outputFileName = std::string(statsFileName);
+        index = outputFileName.find_last_of('.');
+        if(index == std::string::npos) {
+            baseFileName = outputFileName;
+        } else {
+            baseFileName = outputFileName.substr(0, index);
+            suffix = outputFileName.substr(index);
+        }
+        ss << getpid();
+        pid = ss.str();
+        outputFileName = baseFileName + "-" + pid + suffix;
+    }
     eventsFileName = eventsFileName ? eventsFileName : "events.dat";
     plotFileName   = plotFileName   ? plotFileName   : "events.plt";
 
@@ -323,43 +422,31 @@ void kmp_stats_output_module::setupEventColors() {
     return;
 }
 
-void kmp_stats_output_module::printStats(FILE *statsOut, statistic const * theStats, bool areTimers)
+void kmp_stats_output_module::printTimerStats(FILE *statsOut, statistic const * theStats, statistic const * totalStats)
 {
-    if (areTimers)
-    {
-        // Check if we have useful timers, since we don't print zero value timers we need to avoid
-        // printing a header and then no data.
-        bool haveTimers = false;
-        for (int s = 0; s<TIMER_LAST; s++)
-        {
-            if (theStats[s].getCount() != 0)
-            {
-                haveTimers = true;
-                break;
-            }
-        }
-        if (!haveTimers)
-            return;
-    }
+    fprintf (statsOut, "Timer,                      SampleCount,    Min,      Mean,       Max,     Total,        SD\n");
+    for (timer_e s = timer_e(0); s<TIMER_LAST; s = timer_e(s+1)) {
+        statistic const * stat = &theStats[s];
+        char tag = timeStat::noUnits(s) ? ' ' : 'T';
 
-    // Print
-    const char * title = areTimers ? "Timer,                   SampleCount," : "Counter,                 ThreadCount,";
-    fprintf (statsOut, "%s    Min,      Mean,       Max,     Total,        SD\n", title);    
-    if (areTimers) {
-        for (int s = 0; s<TIMER_LAST; s++) {
-            statistic const * stat = &theStats[s];
-            if (stat->getCount() != 0) {
-                char tag = timeStat::noUnits(timer_e(s)) ? ' ' : 'T';
-                fprintf (statsOut, "%-25s, %s\n", timeStat::name(timer_e(s)), stat->format(tag, true).c_str());
-            }
-        }
-    } else {   // Counters
-        for (int s = 0; s<COUNTER_LAST; s++) {
-            statistic const * stat = &theStats[s];
-            fprintf (statsOut, "%-25s, %s\n", counter::name(counter_e(s)), stat->format(' ', true).c_str());
-        }
+        fprintf (statsOut, "%-28s, %s\n", timeStat::name(s), stat->format(tag, true).c_str());
     }
-} 
+    // Also print the Total_ versions of times.
+    for (timer_e s = timer_e(0); s<TIMER_LAST; s = timer_e(s+1)) {
+        char tag = timeStat::noUnits(s) ? ' ' : 'T';
+        if (totalStats && !timeStat::noTotal(s))
+            fprintf(statsOut, "Total_%-22s, %s\n", timeStat::name(s), totalStats[s].format(tag, true).c_str());
+    }
+}
+
+void kmp_stats_output_module::printCounterStats(FILE *statsOut, statistic const * theStats)
+{
+    fprintf (statsOut, "Counter,                 ThreadCount,    Min,      Mean,       Max,     Total,        SD\n");
+    for (int s = 0; s<COUNTER_LAST; s++) {
+        statistic const * stat = &theStats[s];
+        fprintf (statsOut, "%-25s, %s\n", counter::name(counter_e(s)), stat->format(' ', true).c_str());
+    }
+}
 
 void kmp_stats_output_module::printCounters(FILE * statsOut, counter const * theCounters)
 {
@@ -378,10 +465,10 @@ void kmp_stats_output_module::printEvents(FILE* eventsOut, kmp_stats_event_vecto
     for (int i = 0; i < theEvents->size(); i++) {
         kmp_stats_event ev = theEvents->at(i);
         rgb_color color = getEventColor(ev.getTimerName());
-        fprintf(eventsOut, "%d %lu %lu %1.1f rgb(%1.1f,%1.1f,%1.1f) %s\n", 
-                gtid, 
-                ev.getStart(), 
-                ev.getStop(), 
+        fprintf(eventsOut, "%d %lu %lu %1.1f rgb(%1.1f,%1.1f,%1.1f) %s\n",
+                gtid,
+                ev.getStart(),
+                ev.getStop(),
                 1.2 - (ev.getNestLevel() * 0.2),
                 color.r, color.g, color.b,
                 timeStat::name(ev.getTimerName())
@@ -392,20 +479,22 @@ void kmp_stats_output_module::printEvents(FILE* eventsOut, kmp_stats_event_vecto
 
 void kmp_stats_output_module::windupExplicitTimers()
 {
-    // Wind up any explicit timers. We assume that it's fair at this point to just walk all the explcit timers in all threads 
+    // Wind up any explicit timers. We assume that it's fair at this point to just walk all the explcit timers in all threads
     // and say "it's over".
     // If the timer wasn't running, this won't record anything anyway.
     kmp_stats_list::iterator it;
-    for(it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for(it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
+        kmp_stats_list* ptr = *it;
+        ptr->getPartitionedTimers()->windup();
         for (int timer=0; timer<EXPLICIT_TIMER_LAST; timer++) {
-            (*it)->getExplicitTimer(explicit_timer_e(timer))->stop((timer_e)timer);
+            ptr->getExplicitTimer(explicit_timer_e(timer))->stop((timer_e)timer, ptr);
         }
     }
 }
 
 void kmp_stats_output_module::printPloticusFile() {
     int i;
-    int size = __kmp_stats_list.size();
+    int size = __kmp_stats_list->size();
     FILE* plotOut = fopen(plotFileName, "w+");
 
     fprintf(plotOut, "#proc page\n"
@@ -413,7 +502,7 @@ void kmp_stats_output_module::printPloticusFile() {
                      "   scale: 1.0\n\n");
 
     fprintf(plotOut, "#proc getdata\n"
-                     "   file: %s\n\n", 
+                     "   file: %s\n\n",
                      eventsFileName);
 
     fprintf(plotOut, "#proc areadef\n"
@@ -421,7 +510,7 @@ void kmp_stats_output_module::printPloticusFile() {
                      "   titledetails: align=center size=16\n"
                      "   rectangle: 1 1 13 9\n"
                      "   xautorange: datafield=2,3\n"
-                     "   yautorange: -1 %d\n\n", 
+                     "   yautorange: -1 %d\n\n",
                      size);
 
     fprintf(plotOut, "#proc xaxis\n"
@@ -435,7 +524,7 @@ void kmp_stats_output_module::printPloticusFile() {
                      "   stubrange: 0 %d\n"
                      "   stubdetails: size=12\n"
                      "   label: Thread #\n"
-                     "   labeldetails: size=14\n\n", 
+                     "   labeldetails: size=14\n\n",
                      size-1);
 
     fprintf(plotOut, "#proc bars\n"
@@ -466,35 +555,63 @@ void kmp_stats_output_module::printPloticusFile() {
     return;
 }
 
-void kmp_stats_output_module::outputStats(const char* heading) 
+/*
+ * Print some useful information about
+ *    * the date and time this experiment ran.
+ *    * the machine on which it ran.
+ * We output all of this as stylised comments, though we may decide to parse some of it.
+ */
+void kmp_stats_output_module::printHeaderInfo(FILE * statsOut)
 {
-    statistic allStats[TIMER_LAST];
-    statistic allCounters[COUNTER_LAST];
+    std::time_t now = std::time(0);
+    char buffer[40];
+    char hostName[80];
 
-    // stop all the explicit timers for all threads
+    std::strftime(&buffer[0], sizeof(buffer), "%c", std::localtime(&now));
+    fprintf (statsOut, "# Time of run: %s\n", &buffer[0]);
+    if (gethostname(&hostName[0], sizeof(hostName)) == 0)
+        fprintf (statsOut,"# Hostname: %s\n", &hostName[0]);
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+    fprintf (statsOut, "# CPU:  %s\n", &__kmp_cpuinfo.name[0]);
+    fprintf (statsOut, "# Family: %d, Model: %d, Stepping: %d\n", __kmp_cpuinfo.family, __kmp_cpuinfo.model, __kmp_cpuinfo.stepping);
+    if (__kmp_cpuinfo.frequency == 0)
+        fprintf (statsOut, "# Nominal frequency: Unknown\n");
+    else
+        fprintf (statsOut, "# Nominal frequency: %sz\n", formatSI(double(__kmp_cpuinfo.frequency),9,'H').c_str());
+#endif
+}
+
+void kmp_stats_output_module::outputStats(const char* heading)
+{
+    // Stop all the explicit timers in all threads
+    // Do this before declaring the local statistics because thay have constructors so will take time to create.
     windupExplicitTimers();
 
-    FILE * eventsOut;
-    FILE * statsOut = outputFileName ? fopen (outputFileName, "a+") : stderr;
+    statistic allStats[TIMER_LAST];
+    statistic totalStats[TIMER_LAST];           /* Synthesized, cross threads versions of normal timer stats */
+    statistic allCounters[COUNTER_LAST];
 
+    FILE * statsOut = !outputFileName.empty() ? fopen (outputFileName.c_str(), "a+") : stderr;
+    if (!statsOut)
+        statsOut = stderr;
+
+    FILE * eventsOut;
     if (eventPrintingEnabled()) {
         eventsOut = fopen(eventsFileName, "w+");
     }
 
-    if (!statsOut)
-        statsOut = stderr;
-
+    printHeaderInfo (statsOut);
     fprintf(statsOut, "%s\n",heading);
     // Accumulate across threads.
     kmp_stats_list::iterator it;
-    for (it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for (it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
         int t = (*it)->getGtid();
         // Output per thread stats if requested.
-        if (perThreadPrintingEnabled()) {
+        if (printPerThreadFlag) {
             fprintf (statsOut, "Thread %d\n", t);
-            printStats(statsOut, (*it)->getTimers(), true);
-            printCounters(statsOut, (*it)->getCounters());
-            fprintf(statsOut,"\n");
+            printTimerStats (statsOut, (*it)->getTimers(), 0);
+            printCounters   (statsOut, (*it)->getCounters());
+            fprintf (statsOut,"\n");
         }
         // Output per thread events if requested.
         if (eventPrintingEnabled()) {
@@ -502,37 +619,29 @@ void kmp_stats_output_module::outputStats(const char* heading)
             printEvents(eventsOut, &events, t);
         }
 
-        for (int s = 0; s<TIMER_LAST; s++) {
+        // Accumulate timers.
+        for (timer_e s = timer_e(0); s<TIMER_LAST; s = timer_e(s+1)) {
             // See if we should ignore this timer when aggregating
-            if ((timeStat::masterOnly(timer_e(s)) && (t != 0)) || // Timer is only valid on the master and this thread is a worker
-                (timeStat::workerOnly(timer_e(s)) && (t == 0)) || // Timer is only valid on a worker and this thread is the master
-                timeStat::synthesized(timer_e(s))                 // It's a synthesized stat, so there's no raw data for it.
-               )            
+            if ((timeStat::masterOnly(s) && (t != 0)) || // Timer is only valid on the master and this thread is a worker
+                (timeStat::workerOnly(s) && (t == 0))    // Timer is only valid on a worker and this thread is the master
+               )
             {
                 continue;
             }
 
-            statistic * threadStat = (*it)->getTimer(timer_e(s));
+            statistic * threadStat = (*it)->getTimer(s);
             allStats[s] += *threadStat;
+
+            // Add Total stats for timers that are valid in more than one thread
+            if (!timeStat::noTotal(s))
+                totalStats[s].addSample(threadStat->getTotal());
         }
 
-        // Special handling for synthesized statistics.
-        // These just have to be coded specially here for now. 
-        // At present we only have a few: 
-        // The total parallel work done in each thread.
-        // The variance here makes it easy to see load imbalance over the whole program (though, of course,
-        // it's possible to have a code with awful load balance in every parallel region but perfect load
-        // balance oever the whole program.)
-        // The time spent in barriers in each thread.
-        allStats[TIMER_Total_work].addSample ((*it)->getTimer(TIMER_OMP_work)->getTotal());
-
-        // Time in explicit barriers.
-        allStats[TIMER_Total_barrier].addSample ((*it)->getTimer(TIMER_OMP_barrier)->getTotal());
-
-        for (int c = 0; c<COUNTER_LAST; c++) {
-            if (counter::masterOnly(counter_e(c)) && t != 0)
+        // Accumulate counters.
+        for (counter_e c = counter_e(0); c<COUNTER_LAST; c = counter_e(c+1)) {
+            if (counter::masterOnly(c) && t != 0)
                 continue;
-            allCounters[c].addSample ((*it)->getCounter(counter_e(c))->getValue());
+            allCounters[c].addSample ((*it)->getCounter(c)->getValue());
         }
     }
 
@@ -542,13 +651,12 @@ void kmp_stats_output_module::outputStats(const char* heading)
     }
 
     fprintf (statsOut, "Aggregate for all threads\n");
-    printStats (statsOut, &allStats[0], true);
+    printTimerStats (statsOut, &allStats[0], &totalStats[0]);
     fprintf (statsOut, "\n");
-    printStats (statsOut, &allCounters[0], false);
+    printCounterStats (statsOut, &allCounters[0]);
 
     if (statsOut != stderr)
         fclose(statsOut);
-
 }
 
 /* ************************************************** */
@@ -560,7 +668,7 @@ extern "C" {
 void __kmp_reset_stats()
 {
     kmp_stats_list::iterator it;
-    for(it = __kmp_stats_list.begin(); it != __kmp_stats_list.end(); it++) {
+    for(it = __kmp_stats_list->begin(); it != __kmp_stats_list->end(); it++) {
         timeStat * timers     = (*it)->getTimers();
         counter * counters    = (*it)->getCounters();
         explicitTimer * eTimers = (*it)->getExplicitTimers();
@@ -576,17 +684,13 @@ void __kmp_reset_stats()
 
         // reset the event vector so all previous events are "erased"
         (*it)->resetEventVector();
-
-        // May need to restart the explicit timers in thread zero?
     }
-    KMP_START_EXPLICIT_TIMER(OMP_serial);
-    KMP_START_EXPLICIT_TIMER(OMP_start_end);
 }
 
 // This function will reset all stats and stop all threads' explicit timers if they haven't been stopped already.
 void __kmp_output_stats(const char * heading)
 {
-    __kmp_stats_global_output.outputStats(heading);
+    __kmp_stats_global_output->outputStats(heading);
     __kmp_reset_stats();
 }
 
@@ -597,13 +701,23 @@ void __kmp_accumulate_stats_at_exit(void)
         return;
 
     __kmp_output_stats("Statistics on exit");
-    return;
 }
 
-void __kmp_stats_init(void) 
+void __kmp_stats_init(void)
 {
-    return;
+    __kmp_init_tas_lock( & __kmp_stats_lock );
+    __kmp_stats_start_time = tsc_tick_count::now();
+    __kmp_stats_global_output = new kmp_stats_output_module();
+    __kmp_stats_list = new kmp_stats_list();
 }
 
-} // extern "C" 
+void __kmp_stats_fini(void)
+{
+    __kmp_accumulate_stats_at_exit();
+    __kmp_stats_list->deallocate();
+    delete __kmp_stats_global_output;
+    delete __kmp_stats_list;
+}
+
+} // extern "C"
 

runtime/src/kmp_stats.h

@@ -27,6 +27,7 @@
 
 #include <limits>
 #include <math.h>
+#include <vector>
 #include <string>
 #include <stdint.h>
 #include <new> // placement new
@@ -40,16 +41,32 @@
 
 /*!
  * @ingroup STATS_GATHERING
- * \brief flags to describe the statistic ( timers or counter )
+ * \brief flags to describe the statistic (timer or counter)
  *
-*/
-class stats_flags_e {
-    public:
-        const static int onlyInMaster = 1<<0; //!< statistic is valid only for master
-        const static int noUnits      = 1<<1; //!< statistic doesn't need units printed next to it in output
-        const static int synthesized  = 1<<2; //!< statistic's value is created atexit time in the __kmp_output_stats function
-        const static int notInMaster  = 1<<3; //!< statistic is valid for non-master threads
-        const static int logEvent     = 1<<4; //!< statistic can be logged when KMP_STATS_EVENTS is on (valid only for timers)
+ */
+enum stats_flags_e {
+    noTotal      = 1<<0,     //!< do not show a TOTAL_aggregation for this statistic
+    onlyInMaster = 1<<1,     //!< statistic is valid only for master
+    noUnits      = 1<<2,     //!< statistic doesn't need units printed next to it in output
+    notInMaster  = 1<<3,     //!< statistic is valid only for non-master threads
+    logEvent     = 1<<4      //!< statistic can be logged on the event timeline when KMP_STATS_EVENTS is on (valid only for timers)
+};
+
+/*!
+ * @ingroup STATS_GATHERING
+ * \brief the states which a thread can be in
+ *
+ */
+enum stats_state_e {
+    IDLE,
+    SERIAL_REGION,
+    FORK_JOIN_BARRIER,
+    PLAIN_BARRIER,
+    TASKWAIT,
+    TASKYIELD,
+    TASKGROUP,
+    IMPLICIT_TASK,
+    EXPLICIT_TASK
 };
 
 /*!
@@ -67,9 +84,9 @@ class stats_flags_e {
  * Format is "macro(name, flags, arg)"
  *
  * @ingroup STATS_GATHERING
-*/
+ */
 #define KMP_FOREACH_COUNTER(macro, arg)                         \
-    macro (OMP_PARALLEL, stats_flags_e::onlyInMaster, arg)      \
+    macro (OMP_PARALLEL, stats_flags_e::onlyInMaster | stats_flags_e::noTotal, arg) \
     macro (OMP_NESTED_PARALLEL, 0, arg)                         \
     macro (OMP_FOR_static, 0, arg)                              \
     macro (OMP_FOR_dynamic, 0, arg)                             \
@@ -84,15 +101,10 @@ class stats_flags_e {
     macro (REDUCE_wait, 0, arg)                                 \
     macro (REDUCE_nowait, 0, arg)                               \
     macro (OMP_TASKYIELD, 0, arg)                               \
+    macro (OMP_TASKLOOP, 0, arg)                                \
     macro (TASK_executed, 0, arg)                               \
     macro (TASK_cancelled, 0, arg)                              \
-    macro (TASK_stolen, 0, arg)                                 \
-    macro (LAST,0,arg)
-
-// OMP_PARALLEL_args      -- the number of arguments passed to a fork
-// FOR_static_iterations  -- Number of available parallel chunks of work in a static for
-// FOR_dynamic_iterations -- Number of available parallel chunks of work in a dynamic for
-//                           Both adjust for any chunking, so if there were an iteration count of 20 but a chunk size of 10, we'd record 2.
+    macro (TASK_stolen, 0, arg)
 
 /*!
  * \brief Add new timers under KMP_FOREACH_TIMER() macro in kmp_stats.h
@@ -100,7 +112,8 @@ class stats_flags_e {
  * @param macro a user defined macro that takes three arguments - macro(TIMER_NAME, flags, arg)
  * @param arg a user defined argument to send to the user defined macro
  *
- * \details A timer collects multiple samples of some count in each thread and then finally aggregates over all the threads.
+ * \details A timer collects multiple samples of some count in each thread and then finally aggregates alll of the samples from all of the threads.
+ * For most timers the printing code also provides an aggregation over the thread totals. These are printed as TOTAL_foo.
  * The count is normally a time (in ticks), hence the name "timer". (But can be any value, so we use this for "number of arguments passed to fork"
  * as well).
  * For timers the threads are not significant, it's the individual observations that count, so the statistics are at that level.
@@ -108,38 +121,61 @@ class stats_flags_e {
  *
  * @ingroup STATS_GATHERING2
  */
-#define KMP_FOREACH_TIMER(macro, arg)                                   \
-    macro (OMP_start_end, stats_flags_e::onlyInMaster, arg)             \
-    macro (OMP_serial, stats_flags_e::onlyInMaster, arg)                \
-    macro (OMP_work, 0, arg)                                            \
-    macro (Total_work, stats_flags_e::synthesized, arg)                 \
-    macro (OMP_barrier, 0, arg)                                         \
-    macro (Total_barrier, stats_flags_e::synthesized, arg)              \
-    macro (FOR_static_iterations, stats_flags_e::noUnits, arg)          \
-    macro (FOR_static_scheduling, 0, arg)                               \
-    macro (FOR_dynamic_iterations, stats_flags_e::noUnits, arg)         \
-    macro (FOR_dynamic_scheduling, 0, arg)                              \
-    macro (TASK_execution, 0, arg)                                      \
-    macro (OMP_set_numthreads, stats_flags_e::noUnits, arg)             \
-    macro (OMP_PARALLEL_args,  stats_flags_e::noUnits, arg)             \
-    macro (OMP_single, 0, arg)                                          \
-    macro (OMP_master, 0, arg)                                          \
-    KMP_FOREACH_DEVELOPER_TIMER(macro, arg)                             \
-    macro (LAST,0, arg)
+#define KMP_FOREACH_TIMER(macro, arg)                              \
+    macro (OMP_worker_thread_life, stats_flags_e::logEvent, arg)   \
+    macro (FOR_static_scheduling, 0, arg)                          \
+    macro (FOR_dynamic_scheduling, 0, arg)                         \
+    macro (OMP_critical,  0, arg)                                  \
+    macro (OMP_critical_wait,  0, arg)                             \
+    macro (OMP_single,    0, arg)                                  \
+    macro (OMP_master,    0, arg)                                  \
+    macro (OMP_idle, stats_flags_e::logEvent, arg)                 \
+    macro (OMP_plain_barrier, stats_flags_e::logEvent, arg)        \
+    macro (OMP_fork_barrier, stats_flags_e::logEvent, arg)         \
+    macro (OMP_join_barrier, stats_flags_e::logEvent, arg)         \
+    macro (OMP_parallel, stats_flags_e::logEvent, arg)             \
+    macro (OMP_task_immediate, 0, arg)                             \
+    macro (OMP_task_taskwait, 0, arg)                              \
+    macro (OMP_task_taskyield, 0, arg)                             \
+    macro (OMP_task_taskgroup, 0, arg)                             \
+    macro (OMP_task_join_bar, 0, arg)                              \
+    macro (OMP_task_plain_bar, 0, arg)                             \
+    macro (OMP_serial, stats_flags_e::logEvent, arg)               \
+    macro (OMP_taskloop_scheduling, 0, arg)                        \
+    macro (OMP_set_numthreads,    stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
+    macro (OMP_PARALLEL_args,     stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
+    macro (FOR_static_iterations, stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
+    macro (FOR_dynamic_iterations,stats_flags_e::noUnits | stats_flags_e::noTotal, arg) \
+    KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 
 
-// OMP_start_end          -- time from when OpenMP is initialized until the stats are printed at exit
-// OMP_serial             -- thread zero time executing serial code
-// OMP_work               -- elapsed time in code dispatched by a fork (measured in the thread)
-// Total_work             -- a synthesized statistic summarizing how much parallel work each thread executed.
-// OMP_barrier            -- time at "real" barriers
-// Total_barrier          -- a synthesized statistic summarizing how much time at real barriers in each thread
-// FOR_static_scheduling  -- time spent doing scheduling for a static "for"
-// FOR_dynamic_scheduling -- time spent doing scheduling for a dynamic "for"
+// OMP_start_end          -- Time from when OpenMP is initialized until the stats are printed at exit
+// OMP_serial             -- Thread zero time executing serial code
+// OMP_work               -- Elapsed time in code dispatched by a fork (measured in the thread)
+// OMP_barrier            -- Time at "real" barriers (includes task time)
+// FOR_static_scheduling  -- Time spent doing scheduling for a static "for"
+// FOR_dynamic_scheduling -- Time spent doing scheduling for a dynamic "for"
+// OMP_idle               -- Worker threads time spent waiting for inclusion in a parallel region
+// OMP_plain_barrier      -- Time spent in a barrier construct
+// OMP_fork_join_barrier  -- Time spent in a the fork-join barrier surrounding a parallel region
+// OMP_parallel           -- Time spent inside a parallel construct
+// OMP_task_immediate     -- Time spent executing non-deferred tasks
+// OMP_task_taskwait      -- Time spent executing tasks inside a taskwait construct
+// OMP_task_taskyield     -- Time spent executing tasks inside a taskyield construct
+// OMP_task_taskgroup     -- Time spent executing tasks inside a taskygroup construct
+// OMP_task_join_bar      -- Time spent executing tasks inside a join barrier
+// OMP_task_plain_bar     -- Time spent executing tasks inside a barrier construct
+// OMP_single             -- Time spent executing a "single" region
+// OMP_master             -- Time spent executing a "master" region
+// OMP_set_numthreads     -- Values passed to omp_set_num_threads
+// OMP_PARALLEL_args      -- Number of arguments passed to a parallel region
+// FOR_static_iterations  -- Number of available parallel chunks of work in a static for
+// FOR_dynamic_iterations -- Number of available parallel chunks of work in a dynamic for
+//                           Both adjust for any chunking, so if there were an iteration count of 20 but a chunk size of 10, we'd record 2.
 
 #if (KMP_DEVELOPER_STATS)
-// Timers which are of interest tio runtime library developers, not end users.
-// THese have to be explicitly enabled in addition to the other stats.
+// Timers which are of interest to runtime library developers, not end users.
+// These have to be explicitly enabled in addition to the other stats.
 
 // KMP_fork_barrier       -- time in __kmp_fork_barrier
 // KMP_join_barrier       -- time in __kmp_join_barrier
@@ -153,29 +189,23 @@ class stats_flags_e {
 // KMP_tree_release       -- time in __kmp_tree_barrier_release
 // KMP_hyper_gather       -- time in __kmp_hyper_barrier_gather
 // KMP_hyper_release      -- time in __kmp_hyper_barrier_release
-# define KMP_FOREACH_DEVELOPER_TIMER(macro, arg)                        \
-    macro (KMP_fork_call, 0, arg)                                       \
-    macro (KMP_join_call, 0, arg)                                       \
-    macro (KMP_fork_barrier, stats_flags_e::logEvent, arg)              \
-    macro (KMP_join_barrier, stats_flags_e::logEvent, arg)              \
-    macro (KMP_barrier, 0, arg)                                         \
-    macro (KMP_end_split_barrier, 0, arg)                               \
-    macro (KMP_hier_gather, 0, arg)                                     \
-    macro (KMP_hier_release, 0, arg)                                    \
-    macro (KMP_hyper_gather,  stats_flags_e::logEvent, arg)             \
-    macro (KMP_hyper_release,  stats_flags_e::logEvent, arg)            \
-    macro (KMP_linear_gather, 0, arg)                                   \
-    macro (KMP_linear_release, 0, arg)                                  \
-    macro (KMP_tree_gather, 0, arg)                                     \
-    macro (KMP_tree_release, 0, arg)                                    \
-    macro (USER_master_invoke, stats_flags_e::logEvent, arg)            \
-    macro (USER_worker_invoke, stats_flags_e::logEvent, arg)            \
-    macro (USER_resume, stats_flags_e::logEvent, arg)                   \
-    macro (USER_suspend, stats_flags_e::logEvent, arg)                  \
-    macro (USER_launch_thread_loop, stats_flags_e::logEvent, arg)       \
-    macro (KMP_allocate_team, 0, arg)                                   \
-    macro (KMP_setup_icv_copy, 0, arg)                                  \
-    macro (USER_icv_copy, 0, arg)                                       
+# define KMP_FOREACH_DEVELOPER_TIMER(macro, arg) \
+    macro (KMP_fork_call, 0, arg)                \
+    macro (KMP_join_call, 0, arg)                \
+    macro (KMP_end_split_barrier, 0, arg)        \
+    macro (KMP_hier_gather, 0, arg)              \
+    macro (KMP_hier_release, 0, arg)             \
+    macro (KMP_hyper_gather, 0, arg)             \
+    macro (KMP_hyper_release, 0, arg)            \
+    macro (KMP_linear_gather, 0, arg)            \
+    macro (KMP_linear_release, 0, arg)           \
+    macro (KMP_tree_gather, 0, arg)              \
+    macro (KMP_tree_release, 0, arg)             \
+    macro (USER_resume, 0, arg)                  \
+    macro (USER_suspend, 0, arg)                 \
+    macro (KMP_allocate_team, 0, arg)            \
+    macro (KMP_setup_icv_copy, 0, arg)           \
+    macro (USER_icv_copy, 0, arg)
 #else
 # define KMP_FOREACH_DEVELOPER_TIMER(macro, arg)
 #endif
@@ -196,35 +226,41 @@ class stats_flags_e {
  *
  * @ingroup STATS_GATHERING
 */
-#define KMP_FOREACH_EXPLICIT_TIMER(macro, arg)          \
-    macro(OMP_serial, 0, arg)                           \
-    macro(OMP_start_end, 0, arg)                        \
-    macro(OMP_single, 0, arg)                           \
-    macro(OMP_master, 0, arg)                           \
-    KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro,arg)     \
-    macro(LAST, 0, arg)
-
-#if (KMP_DEVELOPER_STATS)
-# define KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro, arg)               \
-    macro(USER_launch_thread_loop, stats_flags_e::logEvent, arg)
-#else
-# define KMP_FOREACH_EXPLICIT_DEVELOPER_TIMER(macro, arg)               
-#endif
+#define KMP_FOREACH_EXPLICIT_TIMER(macro, arg) \
+    KMP_FOREACH_TIMER(macro, arg)
 
 #define ENUMERATE(name,ignore,prefix) prefix##name,
 enum timer_e {
     KMP_FOREACH_TIMER(ENUMERATE, TIMER_)
+    TIMER_LAST
 };
 
 enum explicit_timer_e {
     KMP_FOREACH_EXPLICIT_TIMER(ENUMERATE, EXPLICIT_TIMER_)
+    EXPLICIT_TIMER_LAST
 };
 
 enum counter_e {
     KMP_FOREACH_COUNTER(ENUMERATE, COUNTER_)
+    COUNTER_LAST
 };
 #undef ENUMERATE
 
+class timerPair {
+    explicit_timer_e timer_index;
+    timer_e timer;
+ public:
+    timerPair(explicit_timer_e ti, timer_e t) : timer_index(ti), timer(t) {}
+    inline explicit_timer_e get_index() const { return timer_index; }
+    inline timer_e get_timer() const { return timer; }
+    bool operator==(const timerPair & rhs) {
+        return this->get_index() == rhs.get_index();
+    }
+    bool operator!=(const timerPair & rhs) {
+        return !(*this == rhs);
+    }
+};
+
 class statistic
 {
     double   minVal;
@@ -273,14 +309,13 @@ class timeStat : public statistic
  public:
     timeStat() : statistic() {}
     static const char * name(timer_e e) { return timerInfo[e].name; }
+    static bool  noTotal    (timer_e e) { return timerInfo[e].flags & stats_flags_e::noTotal;      }
     static bool  masterOnly (timer_e e) { return timerInfo[e].flags & stats_flags_e::onlyInMaster; }
     static bool  workerOnly (timer_e e) { return timerInfo[e].flags & stats_flags_e::notInMaster;  }
     static bool  noUnits    (timer_e e) { return timerInfo[e].flags & stats_flags_e::noUnits;      }
-    static bool  synthesized(timer_e e) { return timerInfo[e].flags & stats_flags_e::synthesized;  }
     static bool  logEvent   (timer_e e) { return timerInfo[e].flags & stats_flags_e::logEvent;     }
     static void  clearEventFlags()      {
-        int i;
-        for(i=0;i<TIMER_LAST;i++) {
+        for(int i=0;i<TIMER_LAST;i++) {
             timerInfo[i].flags &= (~(stats_flags_e::logEvent));
         }
     }
@@ -293,15 +328,19 @@ class explicitTimer
 {
     timeStat * stat;
     tsc_tick_count startTime;
+    tsc_tick_count pauseStartTime;
+    tsc_tick_count::tsc_interval_t totalPauseTime;
 
  public:
-    explicitTimer () : stat(0), startTime(0) { }
-    explicitTimer (timeStat * s) : stat(s), startTime() { }
+    explicitTimer () : stat(0), startTime(0), pauseStartTime(0), totalPauseTime() { }
+    explicitTimer (timeStat * s) : stat(s), startTime(), pauseStartTime(0), totalPauseTime() { }
 
     void setStat (timeStat *s) { stat = s; }
     void start(timer_e timerEnumValue);
-    void stop(timer_e timerEnumValue);
-    void reset() { startTime = 0; }
+    void pause() { pauseStartTime = tsc_tick_count::now(); }
+    void resume() { totalPauseTime += (tsc_tick_count::now() - pauseStartTime); }
+    void stop(timer_e timerEnumValue, kmp_stats_list* stats_ptr = nullptr);
+    void reset() { startTime = 0; pauseStartTime = 0; totalPauseTime = 0; }
 };
 
 // Where all you need is to time a block, this is enough.
@@ -314,6 +353,49 @@ class blockTimer : public explicitTimer
     ~blockTimer() { stop(timerEnumValue); }
 };
 
+// Where you need to partition a threads clock ticks into separate states
+// e.g., a partitionedTimers class with two timers of EXECUTING_TASK, and
+//   DOING_NOTHING would render these conditions:
+//   time(EXECUTING_TASK) + time(DOING_NOTHING) = total time thread is alive
+//   No clock tick in the EXECUTING_TASK is a member of DOING_NOTHING and vice versa
+class partitionedTimers
+{
+ private:
+    explicitTimer* timers[EXPLICIT_TIMER_LAST+1];
+    std::vector<timerPair> timer_stack;
+ public:
+    partitionedTimers();
+    void add_timer(explicit_timer_e timer_index, explicitTimer* timer_pointer);
+    void init(timerPair timer_index);
+    void push(timerPair timer_index);
+    void pop();
+    void windup();
+};
+
+// Special wrapper around the partioned timers to aid timing code blocks
+// It avoids the need to have an explicit end, leaving the scope suffices.
+class blockPartitionedTimer
+{
+    partitionedTimers* part_timers;
+    timerPair timer_pair;
+ public:
+    blockPartitionedTimer(partitionedTimers* pt, timerPair tp) : part_timers(pt), timer_pair(tp) { part_timers->push(timer_pair); }
+   ~blockPartitionedTimer() { part_timers->pop(); }
+};
+
+// Special wrapper around the thread state to aid in keeping state in code blocks
+// It avoids the need to have an explicit end, leaving the scope suffices.
+class blockThreadState
+{
+    stats_state_e* state_pointer;
+    stats_state_e  old_state;
+ public:
+    blockThreadState(stats_state_e* thread_state_pointer, stats_state_e new_state) : state_pointer(thread_state_pointer), old_state(*thread_state_pointer) {
+        *state_pointer = new_state;
+    }
+   ~blockThreadState() { *state_pointer = old_state;  }
+};
+
 // If all you want is a count, then you can use this...
 // The individual per-thread counts will be aggregated into a statistic at program exit.
 class counter
@@ -472,14 +554,19 @@ class kmp_stats_list {
     timeStat      _timers[TIMER_LAST+1];
     counter       _counters[COUNTER_LAST+1];
     explicitTimer _explicitTimers[EXPLICIT_TIMER_LAST+1];
+    partitionedTimers _partitionedTimers;
     int           _nestLevel; // one per thread
     kmp_stats_event_vector _event_vector;
     kmp_stats_list* next;
     kmp_stats_list* prev;
+    stats_state_e state;
+    int thread_is_idle_flag;
  public:
-    kmp_stats_list() : next(this) , prev(this) , _event_vector(), _nestLevel(0) {
+    kmp_stats_list() : _nestLevel(0), _event_vector(), next(this), prev(this),
+      state(IDLE), thread_is_idle_flag(0) {
 #define doInit(name,ignore1,ignore2) \
-        getExplicitTimer(EXPLICIT_TIMER_##name)->setStat(getTimer(TIMER_##name));
+        getExplicitTimer(EXPLICIT_TIMER_##name)->setStat(getTimer(TIMER_##name)); \
+        _partitionedTimers.add_timer(EXPLICIT_TIMER_##name, getExplicitTimer(EXPLICIT_TIMER_##name));
         KMP_FOREACH_EXPLICIT_TIMER(doInit,0);
 #undef doInit
     }
@@ -487,6 +574,7 @@ class kmp_stats_list {
     inline timeStat *      getTimer(timer_e idx)                  { return &_timers[idx]; }
     inline counter  *      getCounter(counter_e idx)              { return &_counters[idx]; }
     inline explicitTimer * getExplicitTimer(explicit_timer_e idx) { return &_explicitTimers[idx]; }
+    inline partitionedTimers * getPartitionedTimers()             { return &_partitionedTimers; }
     inline timeStat *      getTimers()                            { return _timers; }
     inline counter  *      getCounters()                          { return _counters; }
     inline explicitTimer * getExplicitTimers()                    { return _explicitTimers; }
@@ -497,6 +585,12 @@ class kmp_stats_list {
     inline void decrementNestValue()                              { _nestLevel--; }
     inline int  getGtid() const                                   { return gtid; }
     inline void setGtid(int newgtid)                              { gtid = newgtid; }
+    inline void setState(stats_state_e newstate)                  { state = newstate; }
+    inline stats_state_e getState() const                         { return state; }
+    inline stats_state_e * getStatePointer()                      { return &state; }
+    inline bool  isIdle()                                         { return thread_is_idle_flag==1; }
+    inline void setIdleFlag()                                     { thread_is_idle_flag = 1; }
+    inline void resetIdleFlag()                                   { thread_is_idle_flag = 0; }
     kmp_stats_list* push_back(int gtid); // returns newly created list node
     inline void     push_event(uint64_t start_time, uint64_t stop_time, int nest_level, timer_e name) {
         _event_vector.push_back(start_time, stop_time, nest_level, name);
@@ -562,7 +656,7 @@ class kmp_stats_output_module {
     };
 
  private:
-    static const char* outputFileName;
+    std::string outputFileName;
     static const char* eventsFileName;
     static const char* plotFileName;
     static int printPerThreadFlag;
@@ -573,19 +667,14 @@ class kmp_stats_output_module {
     void init();
     static void setupEventColors();
     static void printPloticusFile();
-    static void printStats(FILE *statsOut, statistic const * theStats, bool areTimers);
+    static void printHeaderInfo(FILE *statsOut);
+    static void printTimerStats(FILE *statsOut, statistic const * theStats, statistic const * totalStats);
+    static void printCounterStats(FILE *statsOut, statistic const * theStats);
     static void printCounters(FILE * statsOut, counter const * theCounters);
     static void printEvents(FILE * eventsOut, kmp_stats_event_vector* theEvents, int gtid);
     static rgb_color getEventColor(timer_e e) { return timerColorInfo[e]; }
     static void windupExplicitTimers();
-    bool eventPrintingEnabled() {
-        if(printPerThreadEventsFlag) return true;
-        else return false;
-    }
-    bool perThreadPrintingEnabled() {
-        if(printPerThreadFlag) return true;
-        else return false;
-    }
+    bool eventPrintingEnabled() const         { return printPerThreadEventsFlag; }
 
  public:
     kmp_stats_output_module() { init(); }
@@ -596,13 +685,14 @@ class kmp_stats_output_module {
 extern "C" {
 #endif
 void __kmp_stats_init();
+void __kmp_stats_fini();
 void __kmp_reset_stats();
 void __kmp_output_stats(const char *);
 void __kmp_accumulate_stats_at_exit(void);
 // thread local pointer to stats node within list
 extern __thread kmp_stats_list* __kmp_stats_thread_ptr;
 // head to stats list.
-extern kmp_stats_list __kmp_stats_list;
+extern kmp_stats_list* __kmp_stats_list;
 // lock for __kmp_stats_list
 extern kmp_tas_lock_t  __kmp_stats_lock;
 // reference start time
@@ -702,6 +792,35 @@ extern kmp_stats_output_module __kmp_stats_output;
 #define KMP_OUTPUT_STATS(heading_string) \
     __kmp_output_stats(heading_string)
 
+/*!
+ * \brief Initializes the paritioned timers to begin with name.
+ *
+ * @param name timer which you want this thread to begin with
+ *
+ * @ingroup STATS_GATHERING
+*/
+#define KMP_INIT_PARTITIONED_TIMERS(name) \
+    __kmp_stats_thread_ptr->getPartitionedTimers()->init(timerPair(EXPLICIT_TIMER_##name, TIMER_##name))
+
+#define KMP_TIME_PARTITIONED_BLOCK(name) \
+    blockPartitionedTimer __PBLOCKTIME__(__kmp_stats_thread_ptr->getPartitionedTimers(), \
+        timerPair(EXPLICIT_TIMER_##name, TIMER_##name))
+
+#define KMP_PUSH_PARTITIONED_TIMER(name) \
+    __kmp_stats_thread_ptr->getPartitionedTimers()->push(timerPair(EXPLICIT_TIMER_##name, TIMER_##name))
+
+#define KMP_POP_PARTITIONED_TIMER() \
+    __kmp_stats_thread_ptr->getPartitionedTimers()->pop()
+
+#define KMP_SET_THREAD_STATE(state_name) \
+    __kmp_stats_thread_ptr->setState(state_name)
+
+#define KMP_GET_THREAD_STATE() \
+    __kmp_stats_thread_ptr->getState()
+
+#define KMP_SET_THREAD_STATE_BLOCK(state_name) \
+    blockThreadState __BTHREADSTATE__(__kmp_stats_thread_ptr->getStatePointer(), state_name)
+
 /*!
  * \brief resets all stats (counters to 0, timers to 0 elapsed ticks)
  *
@@ -717,6 +836,7 @@ extern kmp_stats_output_module __kmp_stats_output;
 # define KMP_COUNT_DEVELOPER_BLOCK(n)            KMP_COUNT_BLOCK(n)
 # define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   KMP_START_EXPLICIT_TIMER(n)
 # define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    KMP_STOP_EXPLICIT_TIMER(n)
+# define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) KMP_TIME_PARTITIONED_BLOCK(n)
 #else
 // Null definitions
 # define KMP_TIME_DEVELOPER_BLOCK(n)             ((void)0)
@@ -724,6 +844,7 @@ extern kmp_stats_output_module __kmp_stats_output;
 # define KMP_COUNT_DEVELOPER_BLOCK(n)            ((void)0)
 # define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   ((void)0)
 # define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    ((void)0)
+# define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) ((void)0)
 #endif
 
 #else // KMP_STATS_ENABLED
@@ -743,6 +864,14 @@ extern kmp_stats_output_module __kmp_stats_output;
 #define KMP_COUNT_DEVELOPER_BLOCK(n)            ((void)0)
 #define KMP_START_DEVELOPER_EXPLICIT_TIMER(n)   ((void)0)
 #define KMP_STOP_DEVELOPER_EXPLICIT_TIMER(n)    ((void)0)
+#define KMP_INIT_PARTITIONED_TIMERS(name)       ((void)0)
+#define KMP_TIME_PARTITIONED_BLOCK(name)        ((void)0)
+#define KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(n) ((void)0)
+#define KMP_PUSH_PARTITIONED_TIMER(name)        ((void)0)
+#define KMP_POP_PARTITIONED_TIMER()             ((void)0)
+#define KMP_SET_THREAD_STATE(state_name)        ((void)0)
+#define KMP_GET_THREAD_STATE()                  ((void)0)
+#define KMP_SET_THREAD_STATE_BLOCK(state_name)  ((void)0)
 #endif  // KMP_STATS_ENABLED
 
 #endif // KMP_STATS_H

runtime/src/kmp_stats_timing.cpp

@@ -54,14 +54,14 @@ double tsc_tick_count::tick_time()
         char * start = &brand[0];
         for (;*start == ' '; start++)
             ;
-    
+
         char * end = brand + KMP_STRLEN(brand) - 3;
         uint64_t multiplier;
 
         if (*end == 'M') multiplier = 1000LL*1000LL;
         else if (*end == 'G') multiplier = 1000LL*1000LL*1000LL;
         else if (*end == 'T') multiplier = 1000LL*1000LL*1000LL*1000LL;
-        else 
+        else
         {
             cout << "Error determining multiplier '" << *end << "'\n";
             exit (-1);
@@ -69,9 +69,9 @@ double tsc_tick_count::tick_time()
         *end = 0;
         while (*end != ' ') end--;
         end++;
-    
+
         double freq = strtod(end, &start);
-        if (freq == 0.0) 
+        if (freq == 0.0)
         {
             cout << "Error calculating frequency " <<  end << "\n";
             exit (-1);
@@ -95,7 +95,7 @@ std::string formatSI(double interval, int width, char unit)
     if (useSI)
     {
         // Preserve accuracy for small numbers, since we only multiply and the positive powers
-        // of ten are precisely representable. 
+        // of ten are precisely representable.
         static struct { double scale; char prefix; } ranges[] = {
             {1.e12,'f'},
             {1.e9, 'p'},
@@ -112,7 +112,7 @@ std::string formatSI(double interval, int width, char unit)
             {1.e-24,'Z'},
             {1.e-27,'Y'}
         };
-        
+
         if (interval == 0.0)
         {
             os << std::setw(width-3) << std::right << "0.00" << std::setw(3) << unit;
@@ -125,13 +125,13 @@ std::string formatSI(double interval, int width, char unit)
             negative = true;
             interval = -interval;
         }
-        
+
         for (int i=0; i<(int)(sizeof(ranges)/sizeof(ranges[0])); i++)
         {
             if (interval*ranges[i].scale < 1.e0)
             {
                 interval = interval * 1000.e0 * ranges[i].scale;
-                os << std::fixed << std::setprecision(2) << std::setw(width-3) << std::right << 
+                os << std::fixed << std::setprecision(2) << std::setw(width-3) << std::right <<
                     (negative ? -interval : interval) << std::setw(2) << ranges[i].prefix << std::setw(1) << unit;
 
                 return os.str();
@@ -142,27 +142,3 @@ std::string formatSI(double interval, int width, char unit)
 
     return os.str();
 }
-
-tsc_tick_count::tsc_interval_t computeLastInLastOutInterval(timePair * times, int nTimes)
-{
-    timePair lastTimes = times[0];
-    tsc_tick_count * startp = lastTimes.get_startp();
-    tsc_tick_count * endp   = lastTimes.get_endp();
-
-    for (int i=1; i<nTimes; i++)
-    {
-       (*startp) = startp->later(times[i].get_start());
-       (*endp)   = endp->later  (times[i].get_end());
-    }
-
-    return lastTimes.duration();
-}
-
-std::string timePair::format() const
-{
-    std::ostringstream oss;
-
-    oss << start.getValue() << ":" << end.getValue() << " = " << (end-start).getValue();
-
-    return oss.str();
-}

runtime/src/kmp_stats_timing.h

@@ -40,11 +40,16 @@ class tsc_tick_count {
 #endif
         double ticks() const { return double(value); }
         int64_t getValue() const { return value; }
+        tsc_interval_t& operator=(int64_t nvalue) { value = nvalue; return *this; }
 
         friend class tsc_tick_count;
 
-        friend tsc_interval_t operator-(
-        const tsc_tick_count t1, const tsc_tick_count t0);
+        friend tsc_interval_t operator-(const tsc_tick_count& t1,
+                                        const tsc_tick_count& t0);
+        friend tsc_interval_t operator-(const tsc_tick_count::tsc_interval_t& i1,
+                                        const tsc_tick_count::tsc_interval_t& i0);
+        friend tsc_interval_t& operator+=(tsc_tick_count::tsc_interval_t& i1,
+                                         const tsc_tick_count::tsc_interval_t& i0);
     };
 
 #if KMP_HAVE___BUILTIN_READCYCLECOUNTER
@@ -56,26 +61,37 @@ class tsc_tick_count {
 #endif
     tsc_tick_count(int64_t value) : my_count(value) {};
     int64_t getValue() const { return my_count; }
-    tsc_tick_count later (tsc_tick_count const other) const { 
-        return my_count > other.my_count ? (*this) : other; 
+    tsc_tick_count later (tsc_tick_count const other) const {
+        return my_count > other.my_count ? (*this) : other;
     }
-    tsc_tick_count earlier(tsc_tick_count const other) const { 
-        return my_count < other.my_count ? (*this) : other; 
+    tsc_tick_count earlier(tsc_tick_count const other) const {
+        return my_count < other.my_count ? (*this) : other;
     }
 #if KMP_HAVE_TICK_TIME
     static double tick_time(); // returns seconds per cycle (period) of clock
 #endif
     static tsc_tick_count now() { return tsc_tick_count(); } // returns the rdtsc register value
-    friend tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count t1, const tsc_tick_count t0);
+    friend tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count& t1, const tsc_tick_count& t0);
 };
 
-inline tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count t1, const tsc_tick_count t0) 
+inline tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count& t1, const tsc_tick_count& t0)
 {
     return tsc_tick_count::tsc_interval_t( t1.my_count-t0.my_count );
 }
 
+inline tsc_tick_count::tsc_interval_t operator-(const tsc_tick_count::tsc_interval_t& i1, const tsc_tick_count::tsc_interval_t& i0)
+{
+    return tsc_tick_count::tsc_interval_t( i1.value-i0.value );
+}
+
+inline tsc_tick_count::tsc_interval_t& operator+=(tsc_tick_count::tsc_interval_t& i1, const tsc_tick_count::tsc_interval_t& i0)
+{
+    i1.value += i0.value;
+    return i1;
+}
+
 #if KMP_HAVE_TICK_TIME
-inline double tsc_tick_count::tsc_interval_t::seconds() const 
+inline double tsc_tick_count::tsc_interval_t::seconds() const
 {
     return value*tick_time();
 }
@@ -93,27 +109,4 @@ inline std::string formatTicks(double interval, int width)
     return formatSI(interval, width, 'T');
 }
 
-class timePair
-{
-    tsc_tick_count KMP_ALIGN_CACHE start;
-    tsc_tick_count end;
-
-public:
-    timePair() : start(-std::numeric_limits<int64_t>::max()), end(-std::numeric_limits<int64_t>::max()) {}
-    tsc_tick_count get_start() const { return start; }
-    tsc_tick_count get_end()   const { return end; }
-    tsc_tick_count * get_startp()    { return &start; }
-    tsc_tick_count * get_endp()      { return &end; }
-
-    void markStart() { start = tsc_tick_count::now(); }
-    void markEnd()   { end   = tsc_tick_count::now(); }
-    void set_start(tsc_tick_count s) { start = s; }
-    void set_end  (tsc_tick_count e) { end = e; }
-
-    tsc_tick_count::tsc_interval_t duration() const { return end-start; }
-    std::string format() const;
-
-};
-
-extern tsc_tick_count::tsc_interval_t computeLastInLastOutInterval(timePair * times, int nTimes);
 #endif // KMP_STATS_TIMING_H

runtime/src/kmp_str.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_str.c -- String manipulation routines.
+ * kmp_str.cpp -- String manipulation routines.
  */
 
 
@@ -420,8 +420,7 @@ __kmp_str_loc_free(
     kmp_str_loc_t * loc
 ) {
     __kmp_str_fname_free( & loc->fname );
-    KMP_INTERNAL_FREE( loc->_bulk );
-    loc->_bulk = NULL;
+    __kmp_str_free((const char **) &(loc->_bulk));
     loc->file  = NULL;
     loc->func  = NULL;
 } // kmp_str_loc_free
@@ -478,8 +477,6 @@ __kmp_str_eqf(         // True, if strings are equal, false otherwise.
     So standard malloc() is the only available option.
 */
 
-// TODO: Find and replace all regular free() with __kmp_str_free().
-
 char *
 __kmp_str_format(           // Allocated string.
     char const * format,    // Format string.

runtime/src/kmp_stub.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_stub.c -- stub versions of user-callable OpenMP RT functions.
+ * kmp_stub.cpp -- stub versions of user-callable OpenMP RT functions.
  */
 
 
@@ -41,7 +41,9 @@
 #define kmp_set_blocktime            kmpc_set_blocktime
 #define kmp_set_library              kmpc_set_library
 #define kmp_set_defaults             kmpc_set_defaults
+#define kmp_set_disp_num_buffers     kmpc_set_disp_num_buffers
 #define kmp_malloc                   kmpc_malloc
+#define kmp_aligned_malloc           kmpc_aligned_malloc
 #define kmp_calloc                   kmpc_calloc
 #define kmp_realloc                  kmpc_realloc
 #define kmp_free                     kmpc_free
@@ -54,16 +56,16 @@ static size_t __kmps_init() {
     static size_t dummy = 0;
     if ( ! initialized ) {
 
-        // TODO: Analyze KMP_VERSION environment variable, print __kmp_version_copyright and
-        // __kmp_version_build_time.
-        // WARNING: Do not use "fprintf( stderr, ... )" because it will cause unresolved "__iob"
-        // symbol (see C70080). We need to extract __kmp_printf() stuff from kmp_runtime.c and use
-        // it.
+        // TODO: Analyze KMP_VERSION environment variable, print
+        // __kmp_version_copyright and __kmp_version_build_time.
+        // WARNING: Do not use "fprintf( stderr, ... )" because it will cause
+        // unresolved "__iob" symbol (see C70080). We need to extract
+        // __kmp_printf() stuff from kmp_runtime.cpp and use it.
 
-        // Trick with dummy variable forces linker to keep __kmp_version_copyright and
-        // __kmp_version_build_time strings in executable file (in case of static linkage).
-        // When KMP_VERSION analyze is implemented, dummy variable should be deleted, function
-        // should return void.
+        // Trick with dummy variable forces linker to keep __kmp_version_copyright
+        // and __kmp_version_build_time strings in executable file (in case of
+        // static linkage). When KMP_VERSION analysis is implemented, dummy
+        // variable should be deleted, function should return void.
         dummy = __kmp_version_copyright - __kmp_version_build_time;
 
         #if KMP_OS_WINDOWS
@@ -99,9 +101,25 @@ void kmp_set_stacksize_s( size_t arg )    { i; __kmps_set_stacksize( arg ); }
 void kmp_set_blocktime( omp_int_t arg )   { i; __kmps_set_blocktime( arg ); }
 void kmp_set_library( omp_int_t arg )     { i; __kmps_set_library( arg ); }
 void kmp_set_defaults( char const * str ) { i; }
+void kmp_set_disp_num_buffers( omp_int_t arg ) { i; }
 
 /* KMP memory management functions. */
 void * kmp_malloc( size_t size )                 { i; return malloc( size ); }
+void * kmp_aligned_malloc( size_t sz, size_t a ) {
+    i;
+#if KMP_OS_WINDOWS
+    errno = ENOSYS; // not supported
+    return NULL;    // no standard aligned allocator on Windows (pre - C11)
+#else
+    void *res;
+    int err;
+    if( err = posix_memalign( &res, a, sz ) ) {
+        errno = err; // can be EINVAL or ENOMEM
+        return NULL;
+    }
+    return res;
+#endif
+}
 void * kmp_calloc( size_t nelem, size_t elsize ) { i; return calloc( nelem, elsize ); }
 void * kmp_realloc( void *ptr, size_t size )     { i; return realloc( ptr, size ); }
 void   kmp_free( void * ptr )                    { i; free( ptr ); }

runtime/src/kmp_taskdeps.cpp

@@ -73,47 +73,59 @@ __kmp_node_deref ( kmp_info_t *thread, kmp_depnode_t *node )
 static void
 __kmp_depnode_list_free ( kmp_info_t *thread, kmp_depnode_list *list );
 
-static const kmp_int32 kmp_dephash_log2 = 6;
-static const kmp_int32 kmp_dephash_size = (1 << kmp_dephash_log2);
+enum {
+    KMP_DEPHASH_OTHER_SIZE = 97,
+    KMP_DEPHASH_MASTER_SIZE = 997
+};
 
 static inline kmp_int32
-__kmp_dephash_hash ( kmp_intptr_t addr )
+__kmp_dephash_hash ( kmp_intptr_t addr, size_t hsize )
 {
     //TODO alternate to try: set = (((Addr64)(addrUsefulBits * 9.618)) % m_num_sets );
-    return ((addr >> kmp_dephash_log2) ^ addr) % kmp_dephash_size;
+    return ((addr >> 6) ^ (addr >> 2)) % hsize;
 }
 
 static kmp_dephash_t *
-__kmp_dephash_create ( kmp_info_t *thread )
+__kmp_dephash_create ( kmp_info_t *thread, kmp_taskdata_t *current_task )
 {
     kmp_dephash_t *h;
 
-    kmp_int32 size = kmp_dephash_size * sizeof(kmp_dephash_entry_t) + sizeof(kmp_dephash_t);
+    size_t h_size;
+
+    if ( current_task->td_flags.tasktype == TASK_IMPLICIT )
+        h_size = KMP_DEPHASH_MASTER_SIZE;
+    else
+        h_size = KMP_DEPHASH_OTHER_SIZE;
+
+    kmp_int32 size =
+        h_size * sizeof(kmp_dephash_entry_t *) + sizeof(kmp_dephash_t);
 
 #if USE_FAST_MEMORY
     h = (kmp_dephash_t *) __kmp_fast_allocate( thread, size );
 #else
     h = (kmp_dephash_t *) __kmp_thread_malloc( thread, size );
 #endif
+    h->size = h_size;
 
 #ifdef KMP_DEBUG
     h->nelements = 0;
+    h->nconflicts = 0;
 #endif
     h->buckets = (kmp_dephash_entry **)(h+1);
 
-    for ( kmp_int32 i = 0; i < kmp_dephash_size; i++ )
+    for ( size_t i = 0; i < h_size; i++ )
         h->buckets[i] = 0;
 
     return h;
 }
 
-static void
-__kmp_dephash_free ( kmp_info_t *thread, kmp_dephash_t *h )
+void
+__kmp_dephash_free_entries(kmp_info_t *thread, kmp_dephash_t *h)
 {
-    for ( kmp_int32 i=0; i < kmp_dephash_size; i++ ) {
-        if ( h->buckets[i] ) {
+    for (size_t i = 0; i < h->size; i++) {
+        if (h->buckets[i]) {
             kmp_dephash_entry_t *next;
-            for ( kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next ) {
+            for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) {
                 next = entry->next_in_bucket;
                 __kmp_depnode_list_free(thread,entry->last_ins);
                 __kmp_node_deref(thread,entry->last_out);
@@ -123,8 +135,15 @@ __kmp_dephash_free ( kmp_info_t *thread, kmp_dephash_t *h )
                 __kmp_thread_free(thread,entry);
 #endif
             }
+            h->buckets[i] = 0;
         }
     }
+}
+
+void
+__kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h)
+{
+    __kmp_dephash_free_entries(thread, h);
 #if USE_FAST_MEMORY
     __kmp_fast_free(thread,h);
 #else
@@ -135,7 +154,7 @@ __kmp_dephash_free ( kmp_info_t *thread, kmp_dephash_t *h )
 static kmp_dephash_entry *
 __kmp_dephash_find ( kmp_info_t *thread, kmp_dephash_t *h, kmp_intptr_t addr )
 {
-    kmp_int32 bucket = __kmp_dephash_hash(addr);
+    kmp_int32 bucket = __kmp_dephash_hash(addr,h->size);
 
     kmp_dephash_entry_t *entry;
     for ( entry = h->buckets[bucket]; entry; entry = entry->next_in_bucket )
@@ -196,23 +215,41 @@ __kmp_depnode_list_free ( kmp_info_t *thread, kmp_depnode_list *list )
 }
 
 static inline void
-__kmp_track_dependence ( kmp_depnode_t *source, kmp_depnode_t *sink )
+__kmp_track_dependence ( kmp_depnode_t *source, kmp_depnode_t *sink,
+                         kmp_task_t *sink_task )
 {
 #ifdef KMP_SUPPORT_GRAPH_OUTPUT
     kmp_taskdata_t * task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
-    kmp_taskdata_t * task_sink = KMP_TASK_TO_TASKDATA(sink->dn.task);    // this can be NULL when if(0) ...
+    // do not use sink->dn.task as that is only filled after the dependencies
+    // are already processed!
+    kmp_taskdata_t * task_sink = KMP_TASK_TO_TASKDATA(sink_task);
 
     __kmp_printf("%d(%s) -> %d(%s)\n", source->dn.id, task_source->td_ident->psource, sink->dn.id, task_sink->td_ident->psource);
 #endif
+#if OMPT_SUPPORT && OMPT_TRACE
+    /* OMPT tracks dependences between task (a=source, b=sink) in which
+       task a blocks the execution of b through the ompt_new_dependence_callback */
+    if (ompt_enabled &&
+        ompt_callbacks.ompt_callback(ompt_event_task_dependence_pair))
+    {
+        kmp_taskdata_t * task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
+        kmp_taskdata_t * task_sink = KMP_TASK_TO_TASKDATA(sink_task);
+
+        ompt_callbacks.ompt_callback(ompt_event_task_dependence_pair)(
+          task_source->ompt_task_info.task_id,
+          task_sink->ompt_task_info.task_id);
+    }
+#endif /* OMPT_SUPPORT && OMPT_TRACE */
 }
 
 template< bool filter >
 static inline kmp_int32
 __kmp_process_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
-                     bool dep_barrier,kmp_int32 ndeps, kmp_depend_info_t *dep_list)
+                     bool dep_barrier,kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+                     kmp_task_t *task )
 {
-    KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d processing %d depencies : dep_barrier = %d\n", filter, gtid, ndeps, dep_barrier ) );
-    
+    KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d processing %d dependencies : dep_barrier = %d\n", filter, gtid, ndeps, dep_barrier ) );
+
     kmp_info_t *thread = __kmp_threads[ gtid ];
     kmp_int32 npredecessors=0;
     for ( kmp_int32 i = 0; i < ndeps ; i++ ) {
@@ -231,10 +268,10 @@ __kmp_process_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
                 if ( indep->dn.task ) {
                     KMP_ACQUIRE_DEPNODE(gtid,indep);
                     if ( indep->dn.task ) {
-                        __kmp_track_dependence(indep,node);
+                        __kmp_track_dependence(indep,node,task);
                         indep->dn.successors = __kmp_add_node(thread, indep->dn.successors, node);
                         KA_TRACE(40,("__kmp_process_deps<%d>: T#%d adding dependence from %p to %p\n",
-                                 filter,gtid, KMP_TASK_TO_TASKDATA(indep->dn.task), KMP_TASK_TO_TASKDATA(node->dn.task)));
+                                 filter,gtid, KMP_TASK_TO_TASKDATA(indep->dn.task), KMP_TASK_TO_TASKDATA(task)));
                         npredecessors++;
                     }
                     KMP_RELEASE_DEPNODE(gtid,indep);
@@ -247,11 +284,11 @@ __kmp_process_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
         } else if ( last_out && last_out->dn.task ) {
             KMP_ACQUIRE_DEPNODE(gtid,last_out);
             if ( last_out->dn.task ) {
-                __kmp_track_dependence(last_out,node);
+                __kmp_track_dependence(last_out,node,task);
                 last_out->dn.successors = __kmp_add_node(thread, last_out->dn.successors, node);
-                KA_TRACE(40,("__kmp_process_deps<%d>: T#%d adding dependence from %p to %p\n", 
-                             filter,gtid, KMP_TASK_TO_TASKDATA(last_out->dn.task), KMP_TASK_TO_TASKDATA(node->dn.task)));
-                
+                KA_TRACE(40,("__kmp_process_deps<%d>: T#%d adding dependence from %p to %p\n",
+                             filter,gtid, KMP_TASK_TO_TASKDATA(last_out->dn.task), KMP_TASK_TO_TASKDATA(task)));
+
                 npredecessors++;
             }
             KMP_RELEASE_DEPNODE(gtid,last_out);
@@ -312,8 +349,10 @@ __kmp_check_deps ( kmp_int32 gtid, kmp_depnode_t *node, kmp_task_t *task, kmp_de
     // used to pack all npredecessors additions into a single atomic operation at the end
     int npredecessors;
 
-    npredecessors = __kmp_process_deps<true>(gtid, node, hash, dep_barrier, ndeps, dep_list);
-    npredecessors += __kmp_process_deps<false>(gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list);
+    npredecessors = __kmp_process_deps<true>(gtid, node, hash, dep_barrier,
+      ndeps, dep_list, task);
+    npredecessors += __kmp_process_deps<false>(gtid, node, hash, dep_barrier,
+      ndeps_noalias, noalias_dep_list, task);
 
     node->dn.task = task;
     KMP_MB();
@@ -337,14 +376,15 @@ __kmp_release_deps ( kmp_int32 gtid, kmp_taskdata_t *task )
     kmp_depnode_t *node = task->td_depnode;
 
     if ( task->td_dephash ) {
-        KA_TRACE(40, ("__kmp_realease_deps: T#%d freeing dependencies hash of task %p.\n", gtid, task ) );
+        KA_TRACE(40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n", gtid, task ) );
         __kmp_dephash_free(thread,task->td_dephash);
+        task->td_dephash = NULL;
     }
 
     if ( !node ) return;
 
-    KA_TRACE(20, ("__kmp_realease_deps: T#%d notifying succesors of task %p.\n", gtid, task ) );
-    
+    KA_TRACE(20, ("__kmp_release_deps: T#%d notifying successors of task %p.\n", gtid, task ) );
+
     KMP_ACQUIRE_DEPNODE(gtid,node);
     node->dn.task = NULL; // mark this task as finished, so no new dependencies are generated
     KMP_RELEASE_DEPNODE(gtid,node);
@@ -357,8 +397,8 @@ __kmp_release_deps ( kmp_int32 gtid, kmp_taskdata_t *task )
         // successor task can be NULL for wait_depends or because deps are still being processed
         if ( npredecessors == 0 ) {
             KMP_MB();
-            if ( successor->dn.task ) {            
-                KA_TRACE(20, ("__kmp_realease_deps: T#%d successor %p of %p scheduled for execution.\n", gtid, successor->dn.task, task ) );
+            if ( successor->dn.task ) {
+                KA_TRACE(20, ("__kmp_release_deps: T#%d successor %p of %p scheduled for execution.\n", gtid, successor->dn.task, task ) );
                 __kmp_omp_task(gtid,successor->dn.task,false);
             }
         }
@@ -374,7 +414,7 @@ __kmp_release_deps ( kmp_int32 gtid, kmp_taskdata_t *task )
 
     __kmp_node_deref(thread,node);
 
-    KA_TRACE(20, ("__kmp_realease_deps: T#%d all successors of %p notified of completation\n", gtid, task ) );
+    KA_TRACE(20, ("__kmp_release_deps: T#%d all successors of %p notified of completion\n", gtid, task ) );
 }
 
 /*!
@@ -404,15 +444,61 @@ __kmpc_omp_task_with_deps( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_ta
     kmp_info_t *thread = __kmp_threads[ gtid ];
     kmp_taskdata_t * current_task = thread->th.th_current_task;
 
+#if OMPT_SUPPORT && OMPT_TRACE
+    /* OMPT grab all dependences if requested by the tool */
+    if (ompt_enabled && ndeps+ndeps_noalias > 0 &&
+        ompt_callbacks.ompt_callback(ompt_event_task_dependences))
+	{
+        kmp_int32 i;
+
+        new_taskdata->ompt_task_info.ndeps = ndeps+ndeps_noalias;
+        new_taskdata->ompt_task_info.deps = (ompt_task_dependence_t *)
+          KMP_OMPT_DEPS_ALLOC(thread,
+             (ndeps+ndeps_noalias)*sizeof(ompt_task_dependence_t));
+
+        KMP_ASSERT(new_taskdata->ompt_task_info.deps != NULL);
+
+        for (i = 0; i < ndeps; i++)
+        {
+            new_taskdata->ompt_task_info.deps[i].variable_addr =
+              (void*) dep_list[i].base_addr;
+            if (dep_list[i].flags.in && dep_list[i].flags.out)
+                new_taskdata->ompt_task_info.deps[i].dependence_flags =
+                  ompt_task_dependence_type_inout;
+            else if (dep_list[i].flags.out)
+                new_taskdata->ompt_task_info.deps[i].dependence_flags =
+                  ompt_task_dependence_type_out;
+            else if (dep_list[i].flags.in)
+                new_taskdata->ompt_task_info.deps[i].dependence_flags =
+                  ompt_task_dependence_type_in;
+        }
+        for (i = 0; i < ndeps_noalias; i++)
+        {
+            new_taskdata->ompt_task_info.deps[ndeps+i].variable_addr =
+              (void*) noalias_dep_list[i].base_addr;
+            if (noalias_dep_list[i].flags.in && noalias_dep_list[i].flags.out)
+                new_taskdata->ompt_task_info.deps[ndeps+i].dependence_flags =
+                  ompt_task_dependence_type_inout;
+            else if (noalias_dep_list[i].flags.out)
+                new_taskdata->ompt_task_info.deps[ndeps+i].dependence_flags =
+                  ompt_task_dependence_type_out;
+            else if (noalias_dep_list[i].flags.in)
+                new_taskdata->ompt_task_info.deps[ndeps+i].dependence_flags =
+                  ompt_task_dependence_type_in;
+        }
+    }
+#endif /* OMPT_SUPPORT && OMPT_TRACE */
+
     bool serial = current_task->td_flags.team_serial || current_task->td_flags.tasking_ser || current_task->td_flags.final;
-#if OMP_41_ENABLED
-    serial = serial && !(new_taskdata->td_flags.proxy == TASK_PROXY);
+#if OMP_45_ENABLED
+    kmp_task_team_t * task_team = thread->th.th_task_team;
+    serial = serial && !(task_team && task_team->tt.tt_found_proxy_tasks);
 #endif
 
     if ( !serial && ( ndeps > 0 || ndeps_noalias > 0 )) {
         /* if no dependencies have been tracked yet, create the dependence hash */
         if ( current_task->td_dephash == NULL )
-            current_task->td_dephash = __kmp_dephash_create(thread);
+            current_task->td_dephash = __kmp_dephash_create(thread, current_task);
 
 #if USE_FAST_MEMORY
         kmp_depnode_t *node = (kmp_depnode_t *) __kmp_fast_allocate(thread,sizeof(kmp_depnode_t));
@@ -431,14 +517,8 @@ __kmpc_omp_task_with_deps( ident_t *loc_ref, kmp_int32 gtid, kmp_task_t * new_ta
             return TASK_CURRENT_NOT_QUEUED;
         }
     } else {
-#if OMP_41_ENABLED
-        kmp_task_team_t * task_team = thread->th.th_task_team;
-        if ( task_team && task_team->tt.tt_found_proxy_tasks )
-           __kmpc_omp_wait_deps ( loc_ref, gtid, ndeps, dep_list, ndeps_noalias, noalias_dep_list );
-        else
-#endif
-           KA_TRACE(10, ("__kmpc_omp_task_with_deps(exit): T#%d ignored dependencies for task (serialized)"
-                           "loc=%p task=%p\n", gtid, loc_ref, new_taskdata ) );
+        KA_TRACE(10, ("__kmpc_omp_task_with_deps(exit): T#%d ignored dependencies for task (serialized)"
+                      "loc=%p task=%p\n", gtid, loc_ref, new_taskdata ) );
     }
 
     KA_TRACE(10, ("__kmpc_omp_task_with_deps(exit): T#%d task had no blocking dependencies : "
@@ -477,8 +557,8 @@ __kmpc_omp_wait_deps ( ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps, kmp_de
     //   - dependences are not computed in serial teams (except if we have proxy tasks)
     //   - if the dephash is not yet created it means we have nothing to wait for
     bool ignore = current_task->td_flags.team_serial || current_task->td_flags.tasking_ser || current_task->td_flags.final;
-#if OMP_41_ENABLED
-    ignore = ignore && thread->th.th_task_team->tt.tt_found_proxy_tasks == FALSE;
+#if OMP_45_ENABLED
+    ignore = ignore && thread->th.th_task_team != NULL && thread->th.th_task_team->tt.tt_found_proxy_tasks == FALSE;
 #endif
     ignore = ignore || current_task->td_dephash == NULL;
 

runtime/src/kmp_taskq.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_taskq.c -- TASKQ support for OpenMP.
+ * kmp_taskq.cpp -- TASKQ support for OpenMP.
  */
 
 

runtime/src/kmp_threadprivate.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_threadprivate.c -- OpenMP threadprivate support library
+ * kmp_threadprivate.cpp -- OpenMP threadprivate support library
  */
 
 
@@ -519,11 +519,11 @@ kmp_threadprivate_insert( int gtid, void *pc_addr, void *data_addr, size_t pc_si
 /*!
  @ingroup THREADPRIVATE
 
- @param loc source location information 
- @param data  pointer to data being privatized 
- @param ctor  pointer to constructor function for data 
- @param cctor  pointer to copy constructor function for data 
- @param dtor  pointer to destructor function for data 
+ @param loc source location information
+ @param data  pointer to data being privatized
+ @param ctor  pointer to constructor function for data
+ @param cctor  pointer to copy constructor function for data
+ @param dtor  pointer to destructor function for data
 
  Register constructors and destructors for thread private data.
  This function is called when executing in parallel, when we know the thread id.
@@ -617,14 +617,14 @@ __kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data, size_t size
 
 /*!
  @ingroup THREADPRIVATE
- @param loc source location information 
- @param global_tid  global thread number 
- @param data  pointer to data to privatize 
- @param size  size of data to privatize 
- @param cache  pointer to cache 
- @return pointer to private storage 
+ @param loc source location information
+ @param global_tid  global thread number
+ @param data  pointer to data to privatize
+ @param size  size of data to privatize
+ @param cache  pointer to cache
+ @return pointer to private storage
 
- Allocate private storage for threadprivate data. 
+ Allocate private storage for threadprivate data.
 */
 void *
 __kmpc_threadprivate_cached(
@@ -653,7 +653,7 @@ __kmpc_threadprivate_cached(
             // No need to zero the allocated memory; __kmp_allocate does that.
             KC_TRACE( 50, ("__kmpc_threadprivate_cached: T#%d allocated cache at address %p\n",
                            global_tid, my_cache ) );
-            
+
             /* TODO: free all this memory in __kmp_common_destroy using __kmp_threadpriv_cache_list */
             /* Add address of mycache to linked list for cleanup later  */
             kmp_cached_addr_t *tp_cache_addr;
@@ -687,11 +687,11 @@ __kmpc_threadprivate_cached(
 
 /*!
  @ingroup THREADPRIVATE
- @param loc source location information 
- @param data  pointer to data being privatized 
- @param ctor  pointer to constructor function for data 
- @param cctor  pointer to copy constructor function for data 
- @param dtor  pointer to destructor function for data 
+ @param loc source location information
+ @param data  pointer to data being privatized
+ @param ctor  pointer to constructor function for data
+ @param cctor  pointer to copy constructor function for data
+ @param dtor  pointer to destructor function for data
  @param vector_length length of the vector (bytes or elements?)
  Register vector constructors and destructors for thread private data.
 */

runtime/src/kmp_utility.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_utility.c -- Utility routines for the OpenMP support library.
+ * kmp_utility.cpp -- Utility routines for the OpenMP support library.
  */
 
 
@@ -109,12 +109,12 @@ __kmp_parse_frequency(        // R: Frequency in Hz.
 
     double       value  = 0.0;
     char const * unit   = NULL;
-    kmp_uint64   result = ~ 0;
+    kmp_uint64   result = 0;                    /* Zero is a better unknown value than all ones. */
 
     if ( frequency == NULL ) {
         return result;
     }; // if
-    value = strtod( frequency, (char * *) & unit ); // strtod() does not like "char conts *".
+    value = strtod( frequency, (char * *) & unit ); // strtod() does not like "char const *".
     if ( 0 < value && value <= DBL_MAX ) {          // Good value (not overflow, underflow, etc).
         if ( strcmp( unit, "MHz" ) == 0 ) {
             value = value * 1.0E+6;
@@ -279,26 +279,19 @@ __kmp_query_cpuid( kmp_cpuinfo_t *p )
 #endif
     }; // if
 
-    { // Parse CPU brand string for frequency.
-
-        union kmp_cpu_brand_string {
-            struct kmp_cpuid buf[ 3 ];
-            char             string[ sizeof( struct kmp_cpuid ) * 3 + 1 ];
-        }; // union kmp_cpu_brand_string
-        union kmp_cpu_brand_string brand;
+    { // Parse CPU brand string for frequency, saving the string for later.
         int i;
-
-        p->frequency = 0;
+        kmp_cpuid_t * base = (kmp_cpuid_t *)&p->name[0];
 
         // Get CPU brand string.
         for ( i = 0; i < 3; ++ i ) {
-            __kmp_x86_cpuid( 0x80000002 + i, 0, &brand.buf[ i ] );
+            __kmp_x86_cpuid( 0x80000002 + i, 0, base+i );
         }; // for
-        brand.string[ sizeof( brand.string ) - 1 ] = 0; // Just in case. ;-)
-        KA_TRACE( trace_level, ( "cpu brand string: \"%s\"\n", brand.string ) );
+        p->name[ sizeof(p->name) - 1 ] = 0; // Just in case. ;-)
+        KA_TRACE( trace_level, ( "cpu brand string: \"%s\"\n", &p->name[0] ) );
 
         // Parse frequency.
-        p->frequency = __kmp_parse_frequency( strrchr( brand.string, ' ' ) );
+        p->frequency = __kmp_parse_frequency( strrchr( &p->name[0], ' ' ) );
         KA_TRACE( trace_level, ( "cpu frequency from brand string: %" KMP_UINT64_SPEC "\n", p->frequency ) );
     }
 }

runtime/src/kmp_version.cpp

@@ -1,5 +1,5 @@
 /*
- * kmp_version.c
+ * kmp_version.cpp
  */
 
 
@@ -48,6 +48,10 @@
         #define KMP_COMPILER "Intel C++ Compiler 15.0"
     #elif __INTEL_COMPILER == 1600
         #define KMP_COMPILER "Intel C++ Compiler 16.0"
+    #elif __INTEL_COMPILER == 1700
+        #define KMP_COMPILER "Intel C++ Compiler 17.0"
+    #elif __INTEL_COMPILER == 9998
+        #define KMP_COMPILER "Intel C++ Compiler mainline"
     #elif __INTEL_COMPILER == 9999
         #define KMP_COMPILER "Intel C++ Compiler mainline"
     #endif
@@ -85,7 +89,9 @@ int const __kmp_version_major = KMP_VERSION_MAJOR;
 int const __kmp_version_minor = KMP_VERSION_MINOR;
 int const __kmp_version_build = KMP_VERSION_BUILD;
 int const __kmp_openmp_version =
-    #if OMP_40_ENABLED
+    #if OMP_45_ENABLED
+        201511;
+    #elif OMP_40_ENABLED
         201307;
     #else
         201107;
@@ -105,7 +111,7 @@ char const __kmp_version_copyright[]      = KMP_VERSION_PREFIX KMP_COPYRIGHT;
 char const __kmp_version_lib_ver[]        = KMP_VERSION_PREFIX "version: " stringer( KMP_VERSION_MAJOR ) "." stringer( KMP_VERSION_MINOR ) "." stringer( KMP_VERSION_BUILD );
 char const __kmp_version_lib_type[]       = KMP_VERSION_PREFIX "library type: " KMP_LIB_TYPE;
 char const __kmp_version_link_type[]      = KMP_VERSION_PREFIX "link type: " KMP_LINK_TYPE;
-char const __kmp_version_build_time[]     = KMP_VERSION_PREFIX "build time: " __DATE__ " " __TIME__;
+char const __kmp_version_build_time[]     = KMP_VERSION_PREFIX "build time: " "no_timestamp";
 #if KMP_MIC2
     char const __kmp_version_target_env[] = KMP_VERSION_PREFIX "target environment: MIC2";
 #endif
@@ -203,9 +209,6 @@ __kmp_print_version_2( void ) {
         return;
     }; // if
     __kmp_version_2_printed = TRUE;
-
-    #ifndef KMP_STUB
-    #endif // KMP_STUB
 } // __kmp_print_version_2
 
 // end of file //

runtime/src/kmp_wait_release.cpp

@@ -14,13 +14,6 @@
 
 #include "kmp_wait_release.h"
 
-void __kmp_wait_32(kmp_info_t *this_thr, kmp_flag_32 *flag, int final_spin
-                   USE_ITT_BUILD_ARG(void * itt_sync_obj) )
-{
-    __kmp_wait_template(this_thr, flag, final_spin
-                        USE_ITT_BUILD_ARG(itt_sync_obj) );
-}
-
 void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64 *flag, int final_spin
                    USE_ITT_BUILD_ARG(void * itt_sync_obj) )
 {
@@ -28,23 +21,6 @@ void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64 *flag, int final_spin
                         USE_ITT_BUILD_ARG(itt_sync_obj) );
 }
 
-void __kmp_wait_oncore(kmp_info_t *this_thr, kmp_flag_oncore *flag, int final_spin
-                       USE_ITT_BUILD_ARG(void * itt_sync_obj) )
-{
-    __kmp_wait_template(this_thr, flag, final_spin
-                        USE_ITT_BUILD_ARG(itt_sync_obj) );
-}
-
-
-
-void __kmp_release_32(kmp_flag_32 *flag) {
-    __kmp_release_template(flag);
-}
-
 void __kmp_release_64(kmp_flag_64 *flag) {
     __kmp_release_template(flag);
 }
-
-void __kmp_release_oncore(kmp_flag_oncore *flag) {
-    __kmp_release_template(flag);
-}

runtime/src/kmp_wait_release.h

@@ -18,6 +18,7 @@
 
 #include "kmp.h"
 #include "kmp_itt.h"
+#include "kmp_stats.h"
 
 /*!
 @defgroup WAIT_RELEASE Wait/Release operations
@@ -32,7 +33,7 @@ to build higher level operations such as barriers and fork/join.
 @{
 */
 
-/*! 
+/*!
  * The flag_type describes the storage used for the flag.
  */
 enum flag_type {
@@ -83,6 +84,22 @@ class kmp_flag {
     */
 };
 
+#if ! KMP_USE_MONITOR
+# if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
+   // HW TSC is used to reduce overhead (clock tick instead of nanosecond).
+   extern double __kmp_ticks_per_nsec;
+#  define KMP_NOW() __kmp_hardware_timestamp()
+#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC * __kmp_ticks_per_nsec)
+#  define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())
+# else
+   // System time is retrieved sporadically while blocking.
+   extern kmp_uint64 __kmp_now_nsec();
+#  define KMP_NOW() __kmp_now_nsec()
+#  define KMP_BLOCKTIME_INTERVAL() (__kmp_dflt_blocktime * KMP_USEC_PER_SEC)
+#  define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())
+# endif
+#endif
+
 /* Spin wait loop that first does pause, then yield, then sleep. A thread that calls __kmp_wait_*
    must make certain that another thread calls __kmp_release to wake it back up to prevent deadlocks!  */
 template <class C>
@@ -96,6 +113,11 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
     kmp_uint32 hibernate;
     int th_gtid;
     int tasks_completed = FALSE;
+    int oversubscribed;
+#if ! KMP_USE_MONITOR
+    kmp_uint64 poll_count;
+    kmp_uint64 hibernate_goal;
+#endif
 
     KMP_FSYNC_SPIN_INIT(spin, NULL);
     if (flag->done_check()) {
@@ -104,6 +126,9 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
     }
     th_gtid = this_thr->th.th_info.ds.ds_gtid;
     KA_TRACE(20, ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
+#if KMP_STATS_ENABLED
+    stats_state_e thread_state = KMP_GET_THREAD_STATE();
+#endif
 
 #if OMPT_SUPPORT && OMPT_BLAME
     ompt_state_t ompt_state = this_thr->th.ompt_thread_info.state;
@@ -137,6 +162,7 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
     KMP_INIT_YIELD(spins);
 
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
+#if KMP_USE_MONITOR
         // The worker threads cannot rely on the team struct existing at this point.
         // Use the bt values cached in the thread struct instead.
 #ifdef KMP_ADJUST_BLOCKTIME
@@ -160,24 +186,27 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
         KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
                       th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
                       hibernate - __kmp_global.g.g_time.dt.t_value));
+#else
+        hibernate_goal = KMP_NOW() + KMP_BLOCKTIME_INTERVAL();
+        poll_count = 0;
+#endif // KMP_USE_MONITOR
     }
 
+    oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
     KMP_MB();
 
     // Main wait spin loop
     while (flag->notdone_check()) {
         int in_pool;
-
-        /* If the task team is NULL, it means one of things:
-           1) A newly-created thread is first being released by __kmp_fork_barrier(), and
-              its task team has not been set up yet.
-           2) All tasks have been executed to completion, this thread has decremented the task
-              team's ref ct and possibly deallocated it, and should no longer reference it.
-           3) Tasking is off for this region.  This could be because we are in a serialized region
-              (perhaps the outer one), or else tasking was manually disabled (KMP_TASKING=0).  */
         kmp_task_team_t * task_team = NULL;
         if (__kmp_tasking_mode != tskm_immediate_exec) {
             task_team = this_thr->th.th_task_team;
+            /* If the thread's task team pointer is NULL, it means one of 3 things:
+	       1) A newly-created thread is first being released by __kmp_fork_barrier(), and
+	          its task team has not been set up yet.
+	       2) All tasks have been executed to completion.
+	       3) Tasking is off for this region.  This could be because we are in a serialized region
+	          (perhaps the outer one), or else tasking was manually disabled (KMP_TASKING=0).  */
             if (task_team != NULL) {
                 if (TCR_SYNC_4(task_team->tt.tt_active)) {
                     if (KMP_TASKING_ENABLED(task_team))
@@ -199,7 +228,7 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
         }
 
         // If we are oversubscribed, or have waited a bit (and KMP_LIBRARY=throughput), then yield
-        KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
+        KMP_YIELD(oversubscribed);
         // TODO: Should it be number of cores instead of thread contexts? Like:
         // KMP_YIELD(TCR_4(__kmp_nth) > __kmp_ncores);
         // Need performance improvement data to make the change...
@@ -225,6 +254,15 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
             }
         }
 
+#if KMP_STATS_ENABLED
+        // Check if thread has been signalled to idle state
+        // This indicates that the logical "join-barrier" has finished
+        if (this_thr->th.th_stats->isIdle() && KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
+            KMP_SET_THREAD_STATE(IDLE);
+            KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
+        }
+#endif
+
         // Don't suspend if KMP_BLOCKTIME is set to "infinite"
         if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
             continue;
@@ -233,9 +271,14 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
         if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
             continue;
 
+#if KMP_USE_MONITOR
         // If we have waited a bit more, fall asleep
         if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
             continue;
+#else
+        if (KMP_BLOCKING(hibernate_goal, poll_count++))
+            continue;
+#endif
 
         KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
 
@@ -275,6 +318,14 @@ __kmp_wait_template(kmp_info_t *this_thr, C *flag, int final_spin
         }
     }
 #endif
+#if KMP_STATS_ENABLED
+    // If we were put into idle state, pop that off the state stack
+    if (KMP_GET_THREAD_STATE() == IDLE) {
+        KMP_POP_PARTITIONED_TIMER();
+        KMP_SET_THREAD_STATE(thread_state);
+        this_thr->th.th_stats->resetIdleFlag();
+    }
+#endif
 
     KMP_FSYNC_SPIN_ACQUIRED(spin);
 }
@@ -346,16 +397,16 @@ class kmp_basic_flag : public kmp_flag<FlagType> {
  public:
     kmp_basic_flag(volatile FlagType *p) : kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(0) {}
     kmp_basic_flag(volatile FlagType *p, kmp_info_t *thr) : kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(1) {
-        waiting_threads[0] = thr; 
+        waiting_threads[0] = thr;
     }
     kmp_basic_flag(volatile FlagType *p, FlagType c) : kmp_flag<FlagType>(p, traits_type::t), checker(c), num_waiting_threads(0) {}
     /*!
      * param i in   index into waiting_threads
      * @result the thread that is waiting at index i
      */
-    kmp_info_t * get_waiter(kmp_uint32 i) { 
+    kmp_info_t * get_waiter(kmp_uint32 i) {
         KMP_DEBUG_ASSERT(i<num_waiting_threads);
-        return waiting_threads[i]; 
+        return waiting_threads[i];
     }
     /*!
      * @result num_waiting_threads
@@ -366,8 +417,8 @@ class kmp_basic_flag : public kmp_flag<FlagType> {
      *
      * Insert a waiting thread at index 0.
      */
-    void set_waiter(kmp_info_t *thr) { 
-        waiting_threads[0] = thr; 
+    void set_waiter(kmp_info_t *thr) {
+        waiting_threads[0] = thr;
         num_waiting_threads = 1;
     }
     /*!
@@ -398,22 +449,22 @@ class kmp_basic_flag : public kmp_flag<FlagType> {
      * @result Actual flag value before sleep bit(s) set.
      * Notes that there is at least one thread sleeping on the flag by setting sleep bit(s).
      */
-    FlagType set_sleeping() { 
+    FlagType set_sleeping() {
         return traits_type::test_then_or((volatile FlagType *)this->get(), KMP_BARRIER_SLEEP_STATE);
     }
     /*!
      * @result Actual flag value before sleep bit(s) cleared.
      * Notes that there are no longer threads sleeping on the flag by clearing sleep bit(s).
      */
-    FlagType unset_sleeping() { 
+    FlagType unset_sleeping() {
         return traits_type::test_then_and((volatile FlagType *)this->get(), ~KMP_BARRIER_SLEEP_STATE);
     }
-    /*! 
+    /*!
      * @param old_loc in   old value of flag
      * Test whether there are threads sleeping on the flag's old value in old_loc.
      */
     bool is_sleeping_val(FlagType old_loc) { return old_loc & KMP_BARRIER_SLEEP_STATE; }
-    /*! 
+    /*!
      * Test whether there are threads sleeping on the flag.
      */
     bool is_sleeping() { return is_sleeping_val(*(this->get())); }
@@ -558,6 +609,17 @@ public:
     flag_type get_ptr_type() { return flag_oncore; }
 };
 
+// Used to wake up threads, volatile void* flag is usually the th_sleep_loc associated
+// with int gtid.
+static inline void __kmp_null_resume_wrapper(int gtid, volatile void *flag) {
+    if (!flag) return;
+
+    switch (((kmp_flag_64 *)flag)->get_type()) {
+    case flag32: __kmp_resume_32(gtid, NULL); break;
+    case flag64: __kmp_resume_64(gtid, NULL); break;
+    case flag_oncore: __kmp_resume_oncore(gtid, NULL); break;
+    }
+}
 
 /*!
 @}

runtime/src/ompt-event-specific.h

@@ -141,4 +141,12 @@
 
 #define ompt_event_flush_implemented                    ompt_event_UNIMPLEMENTED
 
+#if OMP_40_ENABLED
+# define ompt_event_task_dependences_implemented         ompt_event_MAY_ALWAYS_TRACE
+# define ompt_event_task_dependence_pair_implemented     ompt_event_MAY_ALWAYS_TRACE
+#else
+# define ompt_event_task_dependences_implemented         ompt_event_UNIMPLEMENTED
+# define ompt_event_task_dependence_pair_implemented     ompt_event_UNIMPLEMENTED
+#endif /* OMP_40_ENABLED */
+
 #endif

runtime/src/ompt-general.cpp

@@ -15,7 +15,7 @@
  * ompt include files
  ****************************************************************************/
 
-#include "ompt-specific.c"
+#include "ompt-specific.cpp"
 
 
 
@@ -54,7 +54,7 @@ enum tool_setting_e {
 
 
 typedef void (*ompt_initialize_t) (
-    ompt_function_lookup_t ompt_fn_lookup, 
+    ompt_function_lookup_t ompt_fn_lookup,
     const char *version,
     unsigned int ompt_version
 );
@@ -101,7 +101,7 @@ OMPT_API_ROUTINE ompt_thread_id_t ompt_get_thread_id(void);
  * found, ompt_tool's return value is used to initialize the tool. Otherwise,
  * NULL is returned and OMPT won't be enabled */
 #if OMPT_HAVE_WEAK_ATTRIBUTE
-_OMP_EXTERN 
+_OMP_EXTERN
 __attribute__ (( weak ))
 ompt_initialize_t ompt_tool()
 {
@@ -247,7 +247,7 @@ void ompt_post_init()
     // Initialize the tool if so indicated.
     //--------------------------------------------------
     if (ompt_enabled) {
-        ompt_initialize_fn(ompt_fn_lookup, ompt_get_runtime_version(), 
+        ompt_initialize_fn(ompt_fn_lookup, ompt_get_runtime_version(),
                            OMPT_VERSION);
 
         ompt_thread_t *root_thread = ompt_get_thread();
@@ -432,8 +432,8 @@ OMPT_API_ROUTINE void *ompt_get_task_function(int depth)
  ****************************************************************************/
 
 // Don't define this as static. The loader may choose to eliminate the symbol
-// even though it is needed by tools.  
-#define OMPT_API_PLACEHOLDER 
+// even though it is needed by tools.
+#define OMPT_API_PLACEHOLDER
 
 // Ensure that placeholders don't have mangled names in the symbol table.
 #ifdef __cplusplus
@@ -441,7 +441,7 @@ extern "C" {
 #endif
 
 
-OMPT_API_PLACEHOLDER void ompt_idle(void)  
+OMPT_API_PLACEHOLDER void ompt_idle(void)
 {
     // This function is a placeholder used to represent the calling context of
     // idle OpenMP worker threads. It is not meant to be invoked.

runtime/src/ompt-internal.h

@@ -26,9 +26,13 @@ typedef struct ompt_callbacks_s {
 
 
 typedef struct {
-    ompt_frame_t        frame;
-    void*               function;
-    ompt_task_id_t      task_id;
+    ompt_frame_t            frame;
+    void*                   function;
+    ompt_task_id_t          task_id;
+#if OMP_40_ENABLED
+    int                     ndeps;
+    ompt_task_dependence_t  *deps;
+#endif /* OMP_40_ENABLED */
 } ompt_task_info_t;
 
 
@@ -62,6 +66,16 @@ typedef struct {
 
 extern ompt_callbacks_t ompt_callbacks;
 
+#if OMP_40_ENABLED && OMPT_SUPPORT && OMPT_TRACE
+#if USE_FAST_MEMORY
+#  define KMP_OMPT_DEPS_ALLOC __kmp_fast_allocate
+#  define KMP_OMPT_DEPS_FREE __kmp_fast_free
+# else
+#  define KMP_OMPT_DEPS_ALLOC __kmp_thread_malloc
+#  define KMP_OMPT_DEPS_FREE __kmp_thread_free
+# endif
+#endif /* OMP_40_ENABLED && OMPT_SUPPORT && OMPT_TRACE */
+
 #ifdef __cplusplus
 extern "C" {
 #endif

runtime/src/ompt-specific.cpp

@@ -60,7 +60,12 @@ __ompt_get_teaminfo(int depth, int *size)
 
             // next heavyweight team (if any) after
             // lightweight teams are exhausted
-            if (!lwt && team) team=team->t.t_parent;
+            if (!lwt && team) {
+                team=team->t.t_parent;
+                if (team) {
+                    lwt = LWT_FROM_TEAM(team);
+                }
+            }
 
             depth--;
         }
@@ -257,8 +262,8 @@ __ompt_lw_taskteam_init(ompt_lw_taskteam_t *lwt, kmp_info_t *thr,
     lwt->ompt_team_info.parallel_id = ompt_pid;
     lwt->ompt_team_info.microtask = microtask;
     lwt->ompt_task_info.task_id = 0;
-    lwt->ompt_task_info.frame.reenter_runtime_frame = 0;
-    lwt->ompt_task_info.frame.exit_runtime_frame = 0;
+    lwt->ompt_task_info.frame.reenter_runtime_frame = NULL;
+    lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
     lwt->ompt_task_info.function = NULL;
     lwt->parent = 0;
 }

runtime/src/ompt-specific.h

@@ -69,15 +69,15 @@ ompt_get_thread_gtid(int gtid)
 inline ompt_thread_t *
 ompt_get_thread()
 {
-    int gtid = __kmp_gtid_get_specific();
+    int gtid = __kmp_get_gtid();
     return ompt_get_thread_gtid(gtid);
 }
 
 
-inline void 
+inline void
 ompt_set_thread_state(ompt_thread_t *thread, ompt_state_t state)
 {
-    thread->th.ompt_thread_info.state = state; 
+    thread->th.ompt_thread_info.state = state;
 }
 
 

runtime/src/thirdparty/ittnotify/ittnotify.h

@@ -140,7 +140,7 @@ The same ID may not be reused for different instances, unless a previous
 #  if ITT_PLATFORM==ITT_PLATFORM_WIN
 #    define CDECL __cdecl
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
-#    if defined _M_IX86 || defined __i386__ 
+#    if defined _M_IX86 || defined __i386__
 #      define CDECL __attribute__ ((cdecl))
 #    else  /* _M_IX86 || __i386__ */
 #      define CDECL /* actual only on x86 platform */
@@ -153,7 +153,7 @@ The same ID may not be reused for different instances, unless a previous
 #    define STDCALL __stdcall
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #    if defined _M_IX86 || defined __i386__
-#      define STDCALL __attribute__ ((stdcall)) 
+#      define STDCALL __attribute__ ((stdcall))
 #    else  /* _M_IX86 || __i386__ */
 #      define STDCALL /* supported only on x86 platform */
 #    endif /* _M_IX86 || __i386__ */
@@ -416,19 +416,19 @@ ITT_STUBV(ITTAPI, void, thread_ignore, (void))
  *********************************************************************/
 /** @{ */
 /**
- * @hideinitializer 
+ * @hideinitializer
  * @brief possible value for suppression mask
  */
 #define __itt_suppress_all_errors 0x7fffffff
 
 /**
- * @hideinitializer 
+ * @hideinitializer
  * @brief possible value for suppression mask (suppresses errors from threading analysis)
  */
 #define __itt_suppress_threading_errors 0x000000ff
 
 /**
- * @hideinitializer 
+ * @hideinitializer
  * @brief possible value for suppression mask (suppresses errors from memory analysis)
  */
 #define __itt_suppress_memory_errors 0x0000ff00
@@ -454,7 +454,7 @@ ITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask))
 /** @endcond */
 
 /**
- * @brief Undo the effects of the matching call to __itt_suppress_push  
+ * @brief Undo the effects of the matching call to __itt_suppress_push
  */
 void ITTAPI __itt_suppress_pop(void);
 
@@ -1584,13 +1584,13 @@ ITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void))
  * @brief Specify the type of heap detection/reporting to modify.
  */
 /**
- * @hideinitializer 
+ * @hideinitializer
  * @brief Report on memory leaks.
  */
 #define __itt_heap_leaks 0x00000001
 
 /**
- * @hideinitializer 
+ * @hideinitializer
  * @brief Report on memory growth.
  */
 #define __itt_heap_growth 0x00000002
@@ -1667,7 +1667,7 @@ typedef struct ___itt_domain
  * @ingroup domains
  * @brief Create a domain.
  * Create domain using some domain name: the URI naming style is recommended.
- * Because the set of domains is expected to be static over the application's 
+ * Because the set of domains is expected to be static over the application's
  * execution time, there is no mechanism to destroy a domain.
  * Any domain can be accessed by any thread in the process, regardless of
  * which thread created the domain. This call is thread-safe.
@@ -1801,7 +1801,7 @@ ITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id))
  * @brief Destroy an instance of identifier.
  * This ends the lifetime of the current instance of the given ID value in the trace.
  * Any relationships that are established after this lifetime ends are invalid.
- * This call must be performed before the given ID value can be reused for a different 
+ * This call must be performed before the given ID value can be reused for a different
  * named entity instance.
  * @param[in] domain The domain controlling the execution of this call.
  * @param[in] id The ID to destroy.
@@ -2360,7 +2360,7 @@ ITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id,
  * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task
  * @param[in] key The name of the metadata
  * @param[in] data The metadata itself
- * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated 
+ * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated
 */
 #if ITT_PLATFORM==ITT_PLATFORM_WIN
 void ITTAPI __itt_metadata_str_addA(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length);
@@ -2396,9 +2396,9 @@ ITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id
 #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #else  /* INTEL_NO_ITTNOTIFY_API */
 #if ITT_PLATFORM==ITT_PLATFORM_WIN
-#define __itt_metadata_str_addA(d,x,y,z,a) 
+#define __itt_metadata_str_addA(d,x,y,z,a)
 #define __itt_metadata_str_addA_ptr 0
-#define __itt_metadata_str_addW(d,x,y,z,a) 
+#define __itt_metadata_str_addW(d,x,y,z,a)
 #define __itt_metadata_str_addW_ptr 0
 #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #define __itt_metadata_str_add(d,x,y,z,a)
@@ -2422,7 +2422,7 @@ ITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id
  * @param[in] scope The scope of the instance to which the metadata is to be added
 
  * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task
- 
+
  * @param[in] key The name of the metadata
  * @param[in] type The type of the metadata
  * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added.
@@ -2455,7 +2455,7 @@ ITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __
 
  * @param[in] key The name of the metadata
  * @param[in] data The metadata itself
- * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated 
+ * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated
 */
 #if ITT_PLATFORM==ITT_PLATFORM_WIN
 void ITTAPI __itt_metadata_str_add_with_scopeA(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length);
@@ -2491,9 +2491,9 @@ ITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain
 #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #else  /* INTEL_NO_ITTNOTIFY_API */
 #if ITT_PLATFORM==ITT_PLATFORM_WIN
-#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) 
+#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a)
 #define __itt_metadata_str_add_with_scopeA_ptr  0
-#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) 
+#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a)
 #define __itt_metadata_str_add_with_scopeW_ptr  0
 #else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #define __itt_metadata_str_add_with_scope(d,x,y,z,a)
@@ -3079,9 +3079,9 @@ ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event))
 
 /**
  * @enum __itt_av_data_type
- * @brief Defines types of arrays data (for C/C++ intrinsic types) 
+ * @brief Defines types of arrays data (for C/C++ intrinsic types)
  */
-typedef enum 
+typedef enum
 {
     __itt_e_first = 0,
     __itt_e_char = 0,  /* 1-byte integer */
@@ -3101,8 +3101,8 @@ typedef enum
  * @brief Save an array data to a file.
  * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only).
  * @param[in] data - pointer to the array data
- * @param[in] rank - the rank of the array 
- * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. 
+ * @param[in] rank - the rank of the array
+ * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions.
  * The size of dimensions must be equal to the rank
  * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types)
  * @param[in] filePath - the file path; the output format is defined by the file extension

runtime/src/thirdparty/ittnotify/ittnotify_config.h

@@ -74,7 +74,7 @@
 #  if ITT_PLATFORM==ITT_PLATFORM_WIN
 #    define CDECL __cdecl
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
-#    if defined _M_IX86 || defined __i386__ 
+#    if defined _M_IX86 || defined __i386__
 #      define CDECL __attribute__ ((cdecl))
 #    else  /* _M_IX86 || __i386__ */
 #      define CDECL /* actual only on x86 platform */
@@ -87,7 +87,7 @@
 #    define STDCALL __stdcall
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #    if defined _M_IX86 || defined __i386__
-#      define STDCALL __attribute__ ((stdcall)) 
+#      define STDCALL __attribute__ ((stdcall))
 #    else  /* _M_IX86 || __i386__ */
 #      define STDCALL /* supported only on x86 platform */
 #    endif /* _M_IX86 || __i386__ */
@@ -142,6 +142,14 @@
 #  define ITT_ARCH_PPC64  5
 #endif /* ITT_ARCH_PPC64 */
 
+#ifndef ITT_ARCH_MIPS
+#  define ITT_ARCH_MIPS  6
+#endif /* ITT_ARCH_MIPS */
+
+#ifndef ITT_ARCH_MIPS64
+#  define ITT_ARCH_MIPS64  6
+#endif /* ITT_ARCH_MIPS64 */
+
 
 #ifndef ITT_ARCH
 #  if defined _M_IX86 || defined __i386__
@@ -156,6 +164,10 @@
 #    define ITT_ARCH ITT_ARCH_PPC64
 #  elif defined __aarch64__
 #    define ITT_ARCH ITT_ARCH_AARCH64
+#  elif defined __mips__ && !defined __mips64
+#    define ITT_ARCH ITT_ARCH_MIPS
+#  elif defined __mips__ && defined __mips64
+#    define ITT_ARCH ITT_ARCH_MIPS64
 #  endif
 #endif
 
@@ -293,7 +305,7 @@ ITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend)
                           : "memory");
     return result;
 }
-#elif ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_PPC64 || ITT_ARCH==ITT_ARCH_AARCH64
+#elif ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_PPC64 || ITT_ARCH==ITT_ARCH_AARCH64 || ITT_ARCH==ITT_ARCH_MIPS ||  ITT_ARCH==ITT_ARCH_MIPS64
 #define __TBB_machine_fetchadd4(addr, val) __sync_fetch_and_add(addr, val)
 #endif /* ITT_ARCH==ITT_ARCH_IA64 */
 #ifndef ITT_SIMPLE_INIT

runtime/src/thirdparty/ittnotify/ittnotify_static.c

@@ -23,7 +23,7 @@
 #include <stdarg.h>
 #include <string.h>
 
-#define INTEL_NO_MACRO_BODY 
+#define INTEL_NO_MACRO_BODY
 #define INTEL_ITTNOTIFY_API_PRIVATE
 #include "ittnotify.h"
 #include "legacy/ittnotify.h"
@@ -72,7 +72,7 @@ static const char* ittnotify_lib_name = "libittnotify.dylib";
 
 
 #ifndef LIB_VAR_NAME
-#if ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_ARM
+#if ITT_ARCH==ITT_ARCH_IA32 || ITT_ARCH==ITT_ARCH_ARM || ITT_ARCH==ITT_ARCH_MIPS
 #define LIB_VAR_NAME INTEL_LIBITTNOTIFY32
 #else
 #define LIB_VAR_NAME INTEL_LIBITTNOTIFY64
@@ -285,10 +285,16 @@ ITT_EXTERN_C void _N_(error_handler)(__itt_error_code, va_list args);
 #pragma warning(disable: 4055) /* warning C4055: 'type cast' : from data pointer 'void *' to function pointer 'XXX' */
 #endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 
-static void __itt_report_error(__itt_error_code code, ...)
+static void __itt_report_error(unsigned code_arg, ...)
 {
     va_list args;
-    va_start(args, code);
+    va_start(args, code_arg);
+
+    // We use unsigned for the code argument and explicitly cast it here to the
+    // right enumerator because variadic functions are not compatible with
+    // default promotions.
+    __itt_error_code code = (__itt_error_code)code_arg;
+
     if (_N_(_ittapi_global).error_handler != NULL)
     {
         __itt_error_handler_t* handler = (__itt_error_handler_t*)(size_t)_N_(_ittapi_global).error_handler;

runtime/src/thirdparty/ittnotify/legacy/ittnotify.h

@@ -79,7 +79,7 @@
 #  if ITT_PLATFORM==ITT_PLATFORM_WIN
 #    define CDECL __cdecl
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
-#    if defined _M_IX86 || defined __i386__ 
+#    if defined _M_IX86 || defined __i386__
 #      define CDECL __attribute__ ((cdecl))
 #    else  /* _M_IX86 || __i386__ */
 #      define CDECL /* actual only on x86 platform */
@@ -92,7 +92,7 @@
 #    define STDCALL __stdcall
 #  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
 #    if defined _M_IX86 || defined __i386__
-#      define STDCALL __attribute__ ((stdcall)) 
+#      define STDCALL __attribute__ ((stdcall))
 #    else  /* _M_IX86 || __i386__ */
 #      define STDCALL /* supported only on x86 platform */
 #    endif /* _M_IX86 || __i386__ */

runtime/src/tsan_annotations.cpp

@@ -0,0 +1,63 @@
+/*
+ * tsan_annotations.cpp -- ThreadSanitizer annotations to support data
+ * race detection in OpenMP programs.
+ */
+
+
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tsan_annotations.h"
+
+#include <stdio.h>
+
+typedef unsigned long uptr;
+typedef signed   long sptr;
+
+extern "C" __attribute__((weak)) void AnnotateHappensBefore(const char *f, int l, uptr addr) {}
+extern "C" __attribute__((weak)) void AnnotateHappensAfter(const char *f, int l, uptr addr) {}
+extern "C" __attribute__((weak)) void AnnotateCondVarSignal(const char *f, int l, uptr cv) {}
+extern "C" __attribute__((weak)) void AnnotateCondVarSignalAll(const char *f, int l, uptr cv) {}
+extern "C" __attribute__((weak)) void AnnotateMutexIsNotPHB(const char *f, int l, uptr mu) {}
+extern "C" __attribute__((weak)) void AnnotateCondVarWait(const char *f, int l, uptr cv, uptr lock) {}
+extern "C" __attribute__((weak)) void AnnotateRWLockCreate(const char *f, int l, uptr m) {}
+extern "C" __attribute__((weak)) void AnnotateRWLockCreateStatic(const char *f, int l, uptr m) {}
+extern "C" __attribute__((weak)) void AnnotateRWLockDestroy(const char *f, int l, uptr m) {}
+extern "C" __attribute__((weak)) void AnnotateRWLockAcquired(const char *f, int l, uptr m, uptr is_w) {}
+extern "C" __attribute__((weak)) void AnnotateRWLockReleased(const char *f, int l, uptr m, uptr is_w) {}
+extern "C" __attribute__((weak)) void AnnotateTraceMemory(const char *f, int l, uptr mem) {}
+extern "C" __attribute__((weak)) void AnnotateFlushState(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateNewMemory(const char *f, int l, uptr mem, uptr size) {}
+extern "C" __attribute__((weak)) void AnnotateNoOp(const char *f, int l, uptr mem) {}
+extern "C" __attribute__((weak)) void AnnotateFlushExpectedRaces(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateEnableRaceDetection( const char *f, int l, int enable) {}
+extern "C" __attribute__((weak)) void AnnotateMutexIsUsedAsCondVar( const char *f, int l, uptr mu) {}
+extern "C" __attribute__((weak)) void AnnotatePCQGet( const char *f, int l, uptr pcq) {}
+extern "C" __attribute__((weak)) void AnnotatePCQPut( const char *f, int l, uptr pcq) {}
+extern "C" __attribute__((weak)) void AnnotatePCQDestroy( const char *f, int l, uptr pcq) {}
+extern "C" __attribute__((weak)) void AnnotatePCQCreate( const char *f, int l, uptr pcq) {}
+extern "C" __attribute__((weak)) void AnnotateExpectRace( const char *f, int l, uptr mem, char *desc) {}
+extern "C" __attribute__((weak)) void AnnotateBenignRaceSized( const char *f, int l, uptr mem, uptr size, char *desc) {}
+extern "C" __attribute__((weak)) void AnnotateBenignRace( const char *f, int l, uptr mem, char *desc) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreReadsBegin(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreReadsEnd(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreWritesBegin(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreWritesEnd(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreSyncBegin(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotateIgnoreSyncEnd(const char *f, int l) {}
+extern "C" __attribute__((weak)) void AnnotatePublishMemoryRange( const char *f, int l, uptr addr, uptr size) {}
+extern "C" __attribute__((weak)) void AnnotateUnpublishMemoryRange( const char *f, int l, uptr addr, uptr size) {}
+extern "C" __attribute__((weak)) void AnnotateThreadName( const char *f, int l, char *name) {}
+extern "C" __attribute__((weak)) void WTFAnnotateHappensBefore(const char *f, int l, uptr addr) {}
+extern "C" __attribute__((weak)) void WTFAnnotateHappensAfter(const char *f, int l, uptr addr) {}
+extern "C" __attribute__((weak)) void WTFAnnotateBenignRaceSized( const char *f, int l, uptr mem, uptr sz, char *desc) {}
+extern "C" __attribute__((weak)) int RunningOnValgrind() {return 0;}
+extern "C" __attribute__((weak)) double ValgrindSlowdown(void) {return 0;}
+extern "C" __attribute__((weak)) const char __attribute__((weak))* ThreadSanitizerQuery(const char *query) {return 0;}
+extern "C" __attribute__((weak)) void AnnotateMemoryIsInitialized(const char *f, int l, uptr mem, uptr sz) {}

runtime/src/tsan_annotations.h

@@ -0,0 +1,168 @@
+/*! \file */
+/*
+ * tsan_annotations.h -- ThreadSanitizer annotations to support data
+ * race detection in OpenMP programs.
+ */
+
+
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TSAN_ANNOTATIONS_H
+#define TSAN_ANNOTATIONS_H
+
+#include "kmp_config.h"
+
+/* types as used in tsan/rtl/tsan_interface_ann.cc */
+typedef unsigned long uptr;
+typedef signed   long sptr;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Declaration of all annotation functions in tsan/rtl/tsan_interface_ann.cc */
+void AnnotateHappensBefore(const char *f, int l, uptr addr);
+void AnnotateHappensAfter(const char *f, int l, uptr addr);
+void AnnotateCondVarSignal(const char *f, int l, uptr cv);
+void AnnotateCondVarSignalAll(const char *f, int l, uptr cv);
+void AnnotateMutexIsNotPHB(const char *f, int l, uptr mu);
+void AnnotateCondVarWait(const char *f, int l, uptr cv, uptr lock);
+void AnnotateRWLockCreate(const char *f, int l, uptr m);
+void AnnotateRWLockCreateStatic(const char *f, int l, uptr m);
+void AnnotateRWLockDestroy(const char *f, int l, uptr m);
+void AnnotateRWLockAcquired(const char *f, int l, uptr m, uptr is_w);
+void AnnotateRWLockReleased(const char *f, int l, uptr m, uptr is_w);
+void AnnotateTraceMemory(const char *f, int l, uptr mem);
+void AnnotateFlushState(const char *f, int l);
+void AnnotateNewMemory(const char *f, int l, uptr mem, uptr size);
+void AnnotateNoOp(const char *f, int l, uptr mem);
+void AnnotateFlushExpectedRaces(const char *f, int l);
+void AnnotateEnableRaceDetection( const char *f, int l, int enable);
+void AnnotateMutexIsUsedAsCondVar( const char *f, int l, uptr mu);
+void AnnotatePCQGet( const char *f, int l, uptr pcq);
+void AnnotatePCQPut( const char *f, int l, uptr pcq);
+void AnnotatePCQDestroy( const char *f, int l, uptr pcq);
+void AnnotatePCQCreate( const char *f, int l, uptr pcq);
+void AnnotateExpectRace( const char *f, int l, uptr mem, char *desc);
+void AnnotateBenignRaceSized( const char *f, int l, uptr mem, uptr size, char *desc);
+void AnnotateBenignRace( const char *f, int l, uptr mem, char *desc);
+void AnnotateIgnoreReadsBegin(const char *f, int l);
+void AnnotateIgnoreReadsEnd(const char *f, int l);
+void AnnotateIgnoreWritesBegin(const char *f, int l);
+void AnnotateIgnoreWritesEnd(const char *f, int l);
+void AnnotateIgnoreSyncBegin(const char *f, int l);
+void AnnotateIgnoreSyncEnd(const char *f, int l);
+void AnnotatePublishMemoryRange( const char *f, int l, uptr addr, uptr size);
+void AnnotateUnpublishMemoryRange( const char *f, int l, uptr addr, uptr size);
+void AnnotateThreadName( const char *f, int l, char *name);
+void WTFAnnotateHappensBefore(const char *f, int l, uptr addr);
+void WTFAnnotateHappensAfter(const char *f, int l, uptr addr);
+void WTFAnnotateBenignRaceSized( const char *f, int l, uptr mem, uptr sz, char *desc);
+int RunningOnValgrind();
+double ValgrindSlowdown(void);
+const char * ThreadSanitizerQuery(const char *query);
+void AnnotateMemoryIsInitialized(const char *f, int l, uptr mem, uptr sz);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef TSAN_SUPPORT
+#define ANNOTATE_HAPPENS_AFTER(addr) AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
+#define ANNOTATE_HAPPENS_BEFORE(addr) AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
+#define ANNOTATE_IGNORE_WRITES_BEGIN() AnnotateIgnoreWritesBegin(__FILE__, __LINE__)
+#define ANNOTATE_IGNORE_WRITES_END() AnnotateIgnoreWritesEnd(__FILE__, __LINE__)
+#define ANNOTATE_RWLOCK_CREATE(lck) AnnotateRWLockCreate(__FILE__, __LINE__, (uptr)lck)
+#define ANNOTATE_RWLOCK_RELEASED(lck) AnnotateRWLockAcquired(__FILE__, __LINE__, (uptr)lck, 1)
+#define ANNOTATE_RWLOCK_ACQUIRED(lck) AnnotateRWLockReleased(__FILE__, __LINE__, (uptr)lck, 1)
+
+/* new higher level barrier annotations */
+#define ANNOTATE_NEW_BARRIER_BEGIN(addr) AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
+#define ANNOTATE_NEW_BARRIER_END(addr) AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
+// #define ANNOTATE_NEW_BARRIER_BEGIN(addr)
+// #define ANNOTATE_NEW_BARRIER_END(addr)
+
+
+#define ANNOTATE_REDUCE_AFTER(addr) AnnotateHappensAfter(__FILE__, __LINE__, (uptr)addr)
+#define ANNOTATE_REDUCE_BEFORE(addr) AnnotateHappensBefore(__FILE__, __LINE__, (uptr)addr)
+// #define ANNOTATE_REDUCE_AFTER(addr)
+// #define ANNOTATE_REDUCE_BEFORE(addr)
+
+#else
+#define ANNOTATE_HAPPENS_AFTER(addr)
+#define ANNOTATE_HAPPENS_BEFORE(addr)
+#define ANNOTATE_IGNORE_WRITES_BEGIN()
+#define ANNOTATE_IGNORE_WRITES_END()
+#define ANNOTATE_RWLOCK_CREATE(lck)
+#define ANNOTATE_RWLOCK_RELEASED(lck)
+#define ANNOTATE_RWLOCK_ACQUIRED(lck)
+#define ANNOTATE_NEW_BARRIER_BEGIN(addr)
+#define ANNOTATE_NEW_BARRIER_END(addr)
+#define ANNOTATE_REDUCE_AFTER(addr)
+#define ANNOTATE_REDUCE_BEFORE(addr)
+#endif
+
+#define ANNOTATE_QUEUING
+#define ANNOTATE_TICKET
+#define ANNOTATE_FUTEX
+#define ANNOTATE_TAS
+#define ANNOTATE_DRDPA
+
+#ifdef ANNOTATE_QUEUING
+#define ANNOTATE_QUEUING_CREATE(lck)
+#define ANNOTATE_QUEUING_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
+#define ANNOTATE_QUEUING_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
+#else
+#define ANNOTATE_QUEUING_CREATE(lck)
+#define ANNOTATE_QUEUING_RELEASED(lck)
+#define ANNOTATE_QUEUING_ACQUIRED(lck)
+#endif
+
+#ifdef ANNOTATE_TICKET
+#define ANNOTATE_TICKET_CREATE(lck)
+#define ANNOTATE_TICKET_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
+#define ANNOTATE_TICKET_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
+#else
+#define ANNOTATE_TICKET_CREATE(lck)
+#define ANNOTATE_TICKET_RELEASED(lck)
+#define ANNOTATE_TICKET_ACQUIRED(lck)
+#endif
+
+#ifdef ANNOTATE_FUTEX
+#define ANNOTATE_FUTEX_CREATE(lck)
+#define ANNOTATE_FUTEX_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
+#define ANNOTATE_FUTEX_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
+#else
+#define ANNOTATE_FUTEX_CREATE(lck)
+#define ANNOTATE_FUTEX_RELEASED(lck)
+#define ANNOTATE_FUTEX_ACQUIRED(lck)
+#endif
+
+#ifdef ANNOTATE_TAS
+#define ANNOTATE_TAS_CREATE(lck)
+#define ANNOTATE_TAS_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
+#define ANNOTATE_TAS_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
+#else
+#define ANNOTATE_TAS_CREATE(lck)
+#define ANNOTATE_TAS_RELEASED(lck)
+#define ANNOTATE_TAS_ACQUIRED(lck)
+#endif
+
+#ifdef ANNOTATE_DRDPA
+#define ANNOTATE_DRDPA_CREATE(lck)
+#define ANNOTATE_DRDPA_RELEASED(lck) ANNOTATE_HAPPENS_BEFORE(lck)
+#define ANNOTATE_DRDPA_ACQUIRED(lck) ANNOTATE_HAPPENS_AFTER(lck)
+#else
+#define ANNOTATE_DRDPA_CREATE(lck)
+#define ANNOTATE_DRDPA_RELEASED(lck)
+#define ANNOTATE_DRDPA_ACQUIRED(lck)
+#endif
+
+#endif

runtime/src/z_Linux_asm.s

@@ -109,6 +109,32 @@ KMP_PREFIX_UNDERSCORE(\proc):
 # endif // KMP_OS_DARWIN
 #endif // KMP_ARCH_X86 || KMP_ARCH_x86_64
 
+#if KMP_OS_LINUX && KMP_ARCH_AARCH64
+
+#  define KMP_PREFIX_UNDERSCORE(x) x  // no extra underscore for Linux* OS symbols
+// Format labels so that they don't override function names in gdb's backtraces
+#  define KMP_LABEL(x) .L_##x         // local label hidden from backtraces
+
+.macro ALIGN size
+	.align 1<<(\size)
+.endm
+
+.macro DEBUG_INFO proc
+	.cfi_endproc
+// Not sure why we need .type and .size for the functions
+	ALIGN 2
+	.type  \proc,@function
+	.size  \proc,.-\proc
+.endm
+
+.macro PROC proc
+	ALIGN 2
+	.globl KMP_PREFIX_UNDERSCORE(\proc)
+KMP_PREFIX_UNDERSCORE(\proc):
+	.cfi_startproc
+.endm
+
+#endif // KMP_OS_LINUX && KMP_ARCH_AARCH64
 
 // -----------------------------------------------------------------------
 // data
@@ -1410,11 +1436,347 @@ KMP_LABEL(kmp_1_exit):
 
         DEBUG_INFO __kmp_bsr32
 
-	
+
 // -----------------------------------------------------------------------
 #endif /* KMP_ARCH_X86_64 */
 
-#if KMP_ARCH_ARM
+// '
+#if KMP_OS_LINUX && KMP_ARCH_AARCH64
+
+//------------------------------------------------------------------------
+//
+// typedef void	(*microtask_t)( int *gtid, int *tid, ... );
+//
+// int
+// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
+//		           int gtid, int tid,
+//                         int argc, void *p_argv[] ) {
+//    (*pkfn)( & gtid, & tid, argv[0], ... );
+//    return 1;
+// }
+//
+// parameters:
+//	x0:	pkfn
+//	w1:	gtid
+//	w2:	tid
+//	w3:	argc
+//	x4:	p_argv
+//	x5:	&exit_frame
+//
+// locals:
+//	__gtid:	gtid parm pushed on stack so can pass &gtid to pkfn
+//	__tid:	tid parm pushed on stack so can pass &tid to pkfn
+//
+// reg temps:
+//	 x8:	used to hold pkfn address
+//	 w9:	used as temporary for number of pkfn parms
+//	x10:	used to traverse p_argv array
+//	x11:	used as temporary for stack placement calculation
+//	x12:	used as temporary for stack parameters
+//	x19:	used to preserve exit_frame_ptr, callee-save
+//
+// return:	w0	(always 1/TRUE)
+//
+
+__gtid = 4
+__tid = 8
+
+// -- Begin __kmp_invoke_microtask
+// mark_begin;
+	.text
+	PROC __kmp_invoke_microtask
+
+	stp	x29, x30, [sp, #-16]!
+# if OMPT_SUPPORT
+	stp	x19, x20, [sp, #-16]!
+# endif
+	mov	x29, sp
+
+	orr	w9, wzr, #1
+	add	w9, w9, w3, lsr #1
+	sub	sp, sp, w9, lsl #4
+	mov	x11, sp
+
+	mov	x8, x0
+	str	w1, [x29, #-__gtid]
+	str	w2, [x29, #-__tid]
+	mov	w9, w3
+	mov	x10, x4
+# if OMPT_SUPPORT
+	mov	x19, x5
+	str	x29, [x19]
+# endif
+
+	sub	x0, x29, #__gtid
+	sub	x1, x29, #__tid
+
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x2, [x10]
+
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x3, [x10, #8]!
+
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x4, [x10, #8]!
+
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x5, [x10, #8]!
+
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x6, [x10, #8]!
+
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x7, [x10, #8]!
+
+KMP_LABEL(kmp_0):
+	sub	w9, w9, #1
+	cbz	w9, KMP_LABEL(kmp_1)
+	ldr	x12, [x10, #8]!
+	str	x12, [x11], #8
+	b	KMP_LABEL(kmp_0)
+KMP_LABEL(kmp_1):
+	blr	x8
+	orr	w0, wzr, #1
+	mov	sp, x29
+# if OMPT_SUPPORT
+	str	xzr, [x19]
+	ldp	x19, x20, [sp], #16
+# endif
+	ldp	x29, x30, [sp], #16
+	ret
+
+	DEBUG_INFO __kmp_invoke_microtask
+// -- End  __kmp_invoke_microtask
+
+#endif /* KMP_OS_LINUX && KMP_ARCH_AARCH64 */
+
+#if KMP_ARCH_PPC64
+
+//------------------------------------------------------------------------
+//
+// typedef void	(*microtask_t)( int *gtid, int *tid, ... );
+//
+// int
+// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
+//		           int gtid, int tid,
+//                         int argc, void *p_argv[] ) {
+//    (*pkfn)( & gtid, & tid, argv[0], ... );
+//    return 1;
+// }
+//
+// parameters:
+//	r3:	pkfn
+//	r4:	gtid
+//	r5:	tid
+//	r6:	argc
+//	r7:	p_argv
+//	r8:	&exit_frame
+//
+// return:	r3	(always 1/TRUE)
+//
+	.text
+# if KMP_ARCH_PPC64_LE
+	.abiversion 2
+# endif
+	.globl	__kmp_invoke_microtask
+
+# if KMP_ARCH_PPC64_LE
+	.p2align	4
+# else
+	.p2align	2
+# endif
+
+	.type	__kmp_invoke_microtask,@function
+
+# if KMP_ARCH_PPC64_LE
+__kmp_invoke_microtask:
+.Lfunc_begin0:
+.Lfunc_gep0:
+	addis 2, 12, .TOC.-.Lfunc_gep0@ha
+	addi 2, 2, .TOC.-.Lfunc_gep0@l
+.Lfunc_lep0:
+	.localentry	__kmp_invoke_microtask, .Lfunc_lep0-.Lfunc_gep0
+# else
+	.section	.opd,"aw",@progbits
+__kmp_invoke_microtask:
+	.p2align	3
+	.quad	.Lfunc_begin0
+	.quad	.TOC.@tocbase
+	.quad	0
+	.text
+.Lfunc_begin0:
+# endif
+
+// -- Begin __kmp_invoke_microtask
+// mark_begin;
+
+// We need to allocate a stack frame large enough to hold all of the parameters
+// on the stack for the microtask plus what this function needs. That's 48
+// bytes under the ELFv1 ABI (32 bytes under ELFv2), plus 8*(2 + argc) for the
+// parameters to the microtask, plus 8 bytes to store the values of r4 and r5,
+// and 8 bytes to store r31. With OMP-T support, we need an additional 8 bytes
+// to save r30 to hold a copy of r8.
+
+	.cfi_startproc
+	mflr 0
+	std 31, -8(1)
+	std 0, 16(1)
+
+// This is unusual because normally we'd set r31 equal to r1 after the stack
+// frame is established. In this case, however, we need to dynamically compute
+// the stack frame size, and so we keep a direct copy of r1 to access our
+// register save areas and restore the r1 value before returning.
+	mr 31, 1
+	.cfi_def_cfa_register r31
+	.cfi_offset r31, -8
+	.cfi_offset lr, 16
+
+// Compute the size necessary for the local stack frame.
+# if KMP_ARCH_PPC64_LE
+	li 12, 72
+# else
+	li 12, 88
+# endif
+	sldi 0, 6, 3
+	add 12, 0, 12
+	neg 12, 12
+
+// We need to make sure that the stack frame stays aligned (to 16 bytes, except
+// under the BG/Q CNK, where it must be to 32 bytes).
+# if KMP_OS_CNK
+	li 0, -32
+# else
+	li 0, -16
+# endif
+	and 12, 0, 12
+
+// Establish the local stack frame.
+	stdux 1, 1, 12
+
+# if OMPT_SUPPORT
+	.cfi_offset r30, -16
+	std 30, -16(31)
+	std 1, 0(8)
+	mr 30, 8
+# endif
+
+// Store gtid and tid to the stack because they're passed by reference to the microtask.
+	stw 4, -20(31)
+	stw 5, -24(31)
+
+	mr 12, 6
+	mr 4, 7
+
+	cmpwi 0, 12, 1
+	blt	 0, .Lcall
+
+	ld 5, 0(4)
+
+	cmpwi 0, 12, 2
+	blt	 0, .Lcall
+
+	ld 6, 8(4)
+
+	cmpwi 0, 12, 3
+	blt	 0, .Lcall
+
+	ld 7, 16(4)
+
+	cmpwi 0, 12, 4
+	blt	 0, .Lcall
+
+	ld 8, 24(4)
+
+	cmpwi 0, 12, 5
+	blt	 0, .Lcall
+
+	ld 9, 32(4)
+
+	cmpwi 0, 12, 6
+	blt	 0, .Lcall
+
+	ld 10, 40(4)
+
+	cmpwi 0, 12, 7
+	blt	 0, .Lcall
+
+// There are more than 6 microtask parameters, so we need to store the
+// remainder to the stack.
+	addi 12, 12, -6
+	mtctr 12
+
+// These are set to 8 bytes before the first desired store address (we're using
+// pre-increment loads and stores in the loop below). The parameter save area
+// for the microtask begins 48 + 8*8 == 112 bytes above r1 for ELFv1 and
+// 32 + 8*8 == 96 bytes above r1 for ELFv2.
+	addi 4, 4, 40
+# if KMP_ARCH_PPC64_LE
+	addi 12, 1, 88
+# else
+	addi 12, 1, 104
+# endif
+
+.Lnext:
+	ldu 0, 8(4)
+	stdu 0, 8(12)
+	bdnz .Lnext
+
+.Lcall:
+# if KMP_ARCH_PPC64_LE
+	std 2, 24(1)
+	mr 12, 3
+#else
+	std 2, 40(1)
+// For ELFv1, we need to load the actual function address from the function descriptor.
+	ld 12, 0(3)
+	ld 2, 8(3)
+	ld 11, 16(3)
+#endif
+
+	addi 3, 31, -20
+	addi 4, 31, -24
+
+	mtctr 12
+	bctrl
+# if KMP_ARCH_PPC64_LE
+	ld 2, 24(1)
+# else
+	ld 2, 40(1)
+# endif
+
+# if OMPT_SUPPORT
+	li 3, 0
+	std 3, 0(30)
+# endif
+
+	li 3, 1
+
+# if OMPT_SUPPORT
+	ld 30, -16(31)
+# endif
+
+	mr 1, 31
+	ld 0, 16(1)
+	ld 31, -8(1)
+	mtlr 0
+	blr
+
+	.long	0
+	.quad	0
+.Lfunc_end0:
+	.size	__kmp_invoke_microtask, .Lfunc_end0-.Lfunc_begin0
+	.cfi_endproc
+
+// -- End  __kmp_invoke_microtask
+
+#endif /* KMP_ARCH_PPC64 */
+
+#if KMP_ARCH_ARM || KMP_ARCH_MIPS
     .data
     .comm .gomp_critical_user_,32,8
     .data
@@ -1425,7 +1787,7 @@ __kmp_unnamed_critical_addr:
     .size __kmp_unnamed_critical_addr,4
 #endif /* KMP_ARCH_ARM */
 
-#if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
+#if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
     .data
     .comm .gomp_critical_user_,32,8
     .data

runtime/src/z_Linux_util.cpp

@@ -1,5 +1,5 @@
 /*
- * z_Linux_util.c -- platform specific routines.
+ * z_Linux_util.cpp -- platform specific routines.
  */
 
 
@@ -18,9 +18,11 @@
 #include "kmp_itt.h"
 #include "kmp_str.h"
 #include "kmp_i18n.h"
+#include "kmp_lock.h"
 #include "kmp_io.h"
 #include "kmp_stats.h"
 #include "kmp_wait_release.h"
+#include "kmp_affinity.h"
 
 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD
 # include <alloca.h>
@@ -34,7 +36,7 @@
 
 #if KMP_OS_LINUX && !KMP_OS_CNK
 # include <sys/sysinfo.h>
-# if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+# if KMP_USE_FUTEX
 // We should really include <futex.h>, but that causes compatibility problems on different
 // Linux* OS distributions that either require that you include (or break when you try to include)
 // <pci/types.h>.
@@ -54,11 +56,12 @@
 # include <pthread_np.h>
 #endif
 
-
 #include <dirent.h>
 #include <ctype.h>
 #include <fcntl.h>
 
+#include "tsan_annotations.h"
+
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
@@ -87,6 +90,8 @@ static pthread_mutexattr_t __kmp_suspend_mutex_attr;
 static kmp_cond_align_t    __kmp_wait_cv;
 static kmp_mutex_align_t   __kmp_wait_mx;
 
+double __kmp_ticks_per_nsec;
+
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
@@ -109,118 +114,6 @@ __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
  * Affinity support
  */
 
-/*
- * On some of the older OS's that we build on, these constants aren't present
- * in <asm/unistd.h> #included from <sys.syscall.h>.  They must be the same on
- * all systems of the same arch where they are defined, and they cannot change.
- * stone forever.
- */
-
-#  if KMP_ARCH_X86 || KMP_ARCH_ARM
-#   ifndef __NR_sched_setaffinity
-#    define __NR_sched_setaffinity  241
-#   elif __NR_sched_setaffinity != 241
-#    error Wrong code for setaffinity system call.
-#   endif /* __NR_sched_setaffinity */
-#   ifndef __NR_sched_getaffinity
-#    define __NR_sched_getaffinity  242
-#   elif __NR_sched_getaffinity != 242
-#    error Wrong code for getaffinity system call.
-#   endif /* __NR_sched_getaffinity */
-
-#  elif KMP_ARCH_AARCH64
-#   ifndef __NR_sched_setaffinity
-#    define __NR_sched_setaffinity  122
-#   elif __NR_sched_setaffinity != 122
-#    error Wrong code for setaffinity system call.
-#   endif /* __NR_sched_setaffinity */
-#   ifndef __NR_sched_getaffinity
-#    define __NR_sched_getaffinity  123
-#   elif __NR_sched_getaffinity != 123
-#    error Wrong code for getaffinity system call.
-#   endif /* __NR_sched_getaffinity */
-
-#  elif KMP_ARCH_X86_64
-#   ifndef __NR_sched_setaffinity
-#    define __NR_sched_setaffinity  203
-#   elif __NR_sched_setaffinity != 203
-#    error Wrong code for setaffinity system call.
-#   endif /* __NR_sched_setaffinity */
-#   ifndef __NR_sched_getaffinity
-#    define __NR_sched_getaffinity  204
-#   elif __NR_sched_getaffinity != 204
-#    error Wrong code for getaffinity system call.
-#   endif /* __NR_sched_getaffinity */
-
-#  elif KMP_ARCH_PPC64
-#   ifndef __NR_sched_setaffinity
-#    define __NR_sched_setaffinity  222
-#   elif __NR_sched_setaffinity != 222
-#    error Wrong code for setaffinity system call.
-#   endif /* __NR_sched_setaffinity */
-#   ifndef __NR_sched_getaffinity
-#    define __NR_sched_getaffinity  223
-#   elif __NR_sched_getaffinity != 223
-#    error Wrong code for getaffinity system call.
-#   endif /* __NR_sched_getaffinity */
-
-
-#  else
-#   error Unknown or unsupported architecture
-
-#  endif /* KMP_ARCH_* */
-
-int
-__kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
-{
-    KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
-      "Illegal set affinity operation when not capable");
-#if KMP_USE_HWLOC
-    int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
-#else
-    int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
-#endif
-    if (retval >= 0) {
-        return 0;
-    }
-    int error = errno;
-    if (abort_on_error) {
-        __kmp_msg(
-            kmp_ms_fatal,
-            KMP_MSG( FatalSysError ),
-            KMP_ERR( error ),
-            __kmp_msg_null
-        );
-    }
-    return error;
-}
-
-int
-__kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
-{
-    KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
-      "Illegal get affinity operation when not capable");
-
-#if KMP_USE_HWLOC
-    int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
-#else
-    int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
-#endif
-    if (retval >= 0) {
-        return 0;
-    }
-    int error = errno;
-    if (abort_on_error) {
-        __kmp_msg(
-            kmp_ms_fatal,
-            KMP_MSG( FatalSysError ),
-            KMP_ERR( error ),
-            __kmp_msg_null
-        );
-    }
-    return error;
-}
-
 void
 __kmp_affinity_bind_thread( int which )
 {
@@ -251,8 +144,8 @@ __kmp_affinity_determine_capable(const char *env_var)
 
     int gCode;
     int sCode;
-    kmp_affin_mask_t *buf;
-    buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
+    unsigned char *buf;
+    buf = ( unsigned char * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
 
     // If Linux* OS:
     // If the syscall fails or returns a suggestion for the size,
@@ -272,12 +165,16 @@ __kmp_affinity_determine_capable(const char *env_var)
           && (__kmp_affinity_type != affinity_default)
           && (__kmp_affinity_type != affinity_disabled))) {
             int error = errno;
+            kmp_msg_t err_code = KMP_ERR( error );
             __kmp_msg(
                 kmp_ms_warning,
                 KMP_MSG( GetAffSysCallNotSupported, env_var ),
-                KMP_ERR( error ),
+                err_code,
                 __kmp_msg_null
             );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+                __kmp_str_free(&err_code.str);
+            }
         }
         KMP_AFFINITY_DISABLE();
         KMP_INTERNAL_FREE(buf);
@@ -300,12 +197,16 @@ __kmp_affinity_determine_capable(const char *env_var)
                   && (__kmp_affinity_type != affinity_default)
                   && (__kmp_affinity_type != affinity_disabled))) {
                     int error = errno;
+                    kmp_msg_t err_code = KMP_ERR( error );
                     __kmp_msg(
                         kmp_ms_warning,
                         KMP_MSG( SetAffSysCallNotSupported, env_var ),
-                        KMP_ERR( error ),
+                        err_code,
                         __kmp_msg_null
                     );
+                    if (__kmp_generate_warnings == kmp_warnings_off) {
+                        __kmp_str_free(&err_code.str);
+                    }
                 }
                 KMP_AFFINITY_DISABLE();
                 KMP_INTERNAL_FREE(buf);
@@ -348,12 +249,16 @@ __kmp_affinity_determine_capable(const char *env_var)
                   && (__kmp_affinity_type != affinity_default)
                   && (__kmp_affinity_type != affinity_disabled))) {
                     int error = errno;
+                    kmp_msg_t err_code = KMP_ERR( error );
                     __kmp_msg(
                         kmp_ms_warning,
                         KMP_MSG( GetAffSysCallNotSupported, env_var ),
-                        KMP_ERR( error ),
+                        err_code,
                         __kmp_msg_null
                     );
+                    if (__kmp_generate_warnings == kmp_warnings_off) {
+                        __kmp_str_free(&err_code.str);
+                    }
                 }
                 KMP_AFFINITY_DISABLE();
                 KMP_INTERNAL_FREE(buf);
@@ -379,12 +284,16 @@ __kmp_affinity_determine_capable(const char *env_var)
                   && (__kmp_affinity_type != affinity_default)
                   && (__kmp_affinity_type != affinity_disabled))) {
                     int error = errno;
+                    kmp_msg_t err_code = KMP_ERR( error );
                     __kmp_msg(
                         kmp_ms_warning,
                         KMP_MSG( SetAffSysCallNotSupported, env_var ),
-                        KMP_ERR( error ),
+                        err_code,
                         __kmp_msg_null
                     );
+                    if (__kmp_generate_warnings == kmp_warnings_off) {
+                        __kmp_str_free(&err_code.str);
+                    }
                 }
                 KMP_AFFINITY_DISABLE();
                 KMP_INTERNAL_FREE(buf);
@@ -423,7 +332,7 @@ __kmp_affinity_determine_capable(const char *env_var)
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
 
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
+#if KMP_USE_FUTEX
 
 int
 __kmp_futex_determine_capable()
@@ -440,7 +349,7 @@ __kmp_futex_determine_capable()
     return retval;
 }
 
-#endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
+#endif // KMP_USE_FUTEX
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
@@ -519,7 +428,7 @@ __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
     return old_value;
 }
 
-# if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
+# if KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64)
 kmp_int8
 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
 {
@@ -553,7 +462,7 @@ __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
     }
     return old_value;
 }
-# endif /* KMP_ARCH_X86 */
+# endif /* KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64) */
 
 kmp_int64
 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
@@ -698,6 +607,9 @@ __kmp_launch_worker( void *thr )
 #if KMP_STATS_ENABLED
     // set __thread local index to point to thread-specific stats
     __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
+    KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
+    KMP_SET_THREAD_STATE(IDLE);
+    KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
 #endif
 
 #if USE_ITT_BUILD
@@ -754,7 +666,7 @@ __kmp_launch_worker( void *thr )
     return exit_val;
 }
 
-
+#if KMP_USE_MONITOR
 /* The monitor thread controls all of the threads in the complex */
 
 static void*
@@ -814,13 +726,17 @@ __kmp_launch_monitor( void *thr )
                 rc = sched_setscheduler( 0, sched, & param );
                 if ( rc != 0 ) {
                     int error = errno;
-                  __kmp_msg(
-                      kmp_ms_warning,
-                      KMP_MSG( CantChangeMonitorPriority ),
-                      KMP_ERR( error ),
-                      KMP_MSG( MonitorWillStarve ),
-                      __kmp_msg_null
-                  );
+                    kmp_msg_t err_code = KMP_ERR( error );
+                    __kmp_msg(
+                        kmp_ms_warning,
+                        KMP_MSG( CantChangeMonitorPriority ),
+                        err_code,
+                        KMP_MSG( MonitorWillStarve ),
+                        __kmp_msg_null
+                    );
+                    if (__kmp_generate_warnings == kmp_warnings_off) {
+                        __kmp_str_free(&err_code.str);
+                    }
                 }; // if
             } else {
                 // We cannot abort here, because number of CPUs may be enough for all the threads,
@@ -951,6 +867,7 @@ __kmp_launch_monitor( void *thr )
 
     return thr;
 }
+#endif // KMP_USE_MONITOR
 
 void
 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
@@ -969,13 +886,12 @@ __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
     // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
     // So when thread is created (goes into __kmp_launch_worker) it will
     // set it's __thread local pointer to th->th.th_stats
-    th->th.th_stats = __kmp_stats_list.push_back(gtid);
-    if(KMP_UBER_GTID(gtid)) {
-        __kmp_stats_start_time = tsc_tick_count::now();
-        __kmp_stats_thread_ptr = th->th.th_stats;
-        __kmp_stats_init();
-        KMP_START_EXPLICIT_TIMER(OMP_serial);
-        KMP_START_EXPLICIT_TIMER(OMP_start_end);
+    if(!KMP_UBER_GTID(gtid)) {
+        th->th.th_stats = __kmp_stats_list->push_back(gtid);
+    } else {
+        // For root threads, the __kmp_stats_thread_ptr is set in __kmp_register_root(), so
+        // set the th->th.th_stats field to it.
+        th->th.th_stats = __kmp_stats_thread_ptr;
     }
     __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
 
@@ -994,117 +910,81 @@ __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 
 #ifdef KMP_THREAD_ATTR
-        {
-            status = pthread_attr_init( &thread_attr );
-            if ( status != 0 ) {
-                __kmp_msg(
-                          kmp_ms_fatal,
-                          KMP_MSG( CantInitThreadAttrs ),
-                          KMP_ERR( status ),
-                          __kmp_msg_null
-                          );
-            }; // if
-            status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
-            if ( status != 0 ) {
-                __kmp_msg(
-                          kmp_ms_fatal,
-                          KMP_MSG( CantSetWorkerState ),
-                          KMP_ERR( status ),
-                          __kmp_msg_null
-                          );
-            }; // if
+    status = pthread_attr_init( &thread_attr );
+    if ( status != 0 ) {
+        __kmp_msg(kmp_ms_fatal, KMP_MSG( CantInitThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
+    }; // if
+    status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
+    if ( status != 0 ) {
+        __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerState ), KMP_ERR( status ), __kmp_msg_null);
+    }; // if
 
-            /* Set stack size for this thread now. 
-             * The multiple of 2 is there because on some machines, requesting an unusual stacksize
-             * causes the thread to have an offset before the dummy alloca() takes place to create the
-             * offset.  Since we want the user to have a sufficient stacksize AND support a stack offset, we 
-             * alloca() twice the offset so that the upcoming alloca() does not eliminate any premade
-             * offset, and also gives the user the stack space they requested for all threads */
-            stack_size += gtid * __kmp_stkoffset * 2;
+    /* Set stack size for this thread now.
+     * The multiple of 2 is there because on some machines, requesting an unusual stacksize
+     * causes the thread to have an offset before the dummy alloca() takes place to create the
+     * offset.  Since we want the user to have a sufficient stacksize AND support a stack offset, we
+     * alloca() twice the offset so that the upcoming alloca() does not eliminate any premade
+     * offset, and also gives the user the stack space they requested for all threads */
+    stack_size += gtid * __kmp_stkoffset * 2;
 
-            KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
-                            "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
-                            gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
+    KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
+                    "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
+                    gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
 
 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
-                status = pthread_attr_setstacksize( & thread_attr, stack_size );
+    status = pthread_attr_setstacksize( & thread_attr, stack_size );
 #  ifdef KMP_BACKUP_STKSIZE
-            if ( status != 0 ) {
-                if ( ! __kmp_env_stksize ) {
-                    stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
-                    __kmp_stksize = KMP_BACKUP_STKSIZE;
-                    KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
-                                   "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
-                                   "bytes\n",
-                                   gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
-                              );
-                    status = pthread_attr_setstacksize( &thread_attr, stack_size );
-                }; // if
-            }; // if
+    if ( status != 0 ) {
+        if ( ! __kmp_env_stksize ) {
+            stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
+            __kmp_stksize = KMP_BACKUP_STKSIZE;
+            KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
+                           "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
+                           "bytes\n",
+                           gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
+                      );
+            status = pthread_attr_setstacksize( &thread_attr, stack_size );
+        }; // if
+    }; // if
 #  endif /* KMP_BACKUP_STKSIZE */
-            if ( status != 0 ) {
-                __kmp_msg(
-                          kmp_ms_fatal,
-                          KMP_MSG( CantSetWorkerStackSize, stack_size ),
-                          KMP_ERR( status ),
-                          KMP_HNT( ChangeWorkerStackSize  ),
-                          __kmp_msg_null
-                          );
-            }; // if
+    if ( status != 0 ) {
+        __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
+                  KMP_HNT( ChangeWorkerStackSize  ), __kmp_msg_null);
+    }; // if
 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
-        }
+
 #endif /* KMP_THREAD_ATTR */
 
-        {
-            status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
-            if ( status != 0 || ! handle ) { // ??? Why do we check handle??
+    status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
+    if ( status != 0 || ! handle ) { // ??? Why do we check handle??
 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
-                if ( status == EINVAL ) {
-                    __kmp_msg(
-                              kmp_ms_fatal,
-                              KMP_MSG( CantSetWorkerStackSize, stack_size ),
-                              KMP_ERR( status ),
-                              KMP_HNT( IncreaseWorkerStackSize ),
-                              __kmp_msg_null
-                              );
-                };
-                if ( status == ENOMEM ) {
-                    __kmp_msg(
-                              kmp_ms_fatal,
-                              KMP_MSG( CantSetWorkerStackSize, stack_size ),
-                              KMP_ERR( status ),
-                              KMP_HNT( DecreaseWorkerStackSize ),
-                              __kmp_msg_null
-                              );
-                };
+        if ( status == EINVAL ) {
+            __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
+                      KMP_HNT( IncreaseWorkerStackSize ), __kmp_msg_null);
+        };
+        if ( status == ENOMEM ) {
+            __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
+                      KMP_HNT( DecreaseWorkerStackSize ), __kmp_msg_null);
+        };
 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
-                if ( status == EAGAIN ) {
-                    __kmp_msg(
-                              kmp_ms_fatal,
-                              KMP_MSG( NoResourcesForWorkerThread ),
-                              KMP_ERR( status ),
-                              KMP_HNT( Decrease_NUM_THREADS ),
-                              __kmp_msg_null
-                              );
-                }; // if
-                KMP_SYSFAIL( "pthread_create", status );
-            }; // if
+        if ( status == EAGAIN ) {
+            __kmp_msg(kmp_ms_fatal, KMP_MSG( NoResourcesForWorkerThread ), KMP_ERR( status ),
+                      KMP_HNT( Decrease_NUM_THREADS ), __kmp_msg_null);
+        }; // if
+        KMP_SYSFAIL( "pthread_create", status );
+    }; // if
 
-            th->th.th_info.ds.ds_thread = handle;
-        }
+    th->th.th_info.ds.ds_thread = handle;
 
 #ifdef KMP_THREAD_ATTR
-        {
-            status = pthread_attr_destroy( & thread_attr );
-            if ( status ) {
-                __kmp_msg(
-                          kmp_ms_warning,
-                          KMP_MSG( CantDestroyThreadAttrs ),
-                          KMP_ERR( status ),
-                          __kmp_msg_null
-                          );
-            }; // if
+    status = pthread_attr_destroy( & thread_attr );
+    if ( status ) {
+        kmp_msg_t err_code = KMP_ERR( status );
+        __kmp_msg(kmp_ms_warning, KMP_MSG( CantDestroyThreadAttrs ), err_code, __kmp_msg_null);
+        if (__kmp_generate_warnings == kmp_warnings_off) {
+            __kmp_str_free(&err_code.str);
         }
+    }; // if
 #endif /* KMP_THREAD_ATTR */
 
     KMP_MB();       /* Flush all pending memory write invalidates.  */
@@ -1114,6 +994,7 @@ __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
 } // __kmp_create_worker
 
 
+#if KMP_USE_MONITOR
 void
 __kmp_create_monitor( kmp_info_t *th )
 {
@@ -1198,13 +1079,17 @@ __kmp_create_monitor( kmp_info_t *th )
                 __kmp_monitor_stksize *= 2;
                 goto retry;
             }
+            kmp_msg_t err_code = KMP_ERR( status );
             __kmp_msg(
                 kmp_ms_warning,  // should this be fatal?  BB
                 KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
-                KMP_ERR( status ),
+                err_code,
                 KMP_HNT( ChangeMonitorStackSize ),
                 __kmp_msg_null
             );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+                __kmp_str_free(&err_code.str);
+            }
         }; // if
     #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
 
@@ -1260,12 +1145,16 @@ __kmp_create_monitor( kmp_info_t *th )
     #ifdef KMP_THREAD_ATTR
         status = pthread_attr_destroy( & thread_attr );
         if ( status != 0 ) {
-            __kmp_msg(    //
+            kmp_msg_t err_code = KMP_ERR( status );
+            __kmp_msg( 
                 kmp_ms_warning,
                 KMP_MSG( CantDestroyThreadAttrs ),
-                KMP_ERR( status ),
+                err_code,
                 __kmp_msg_null
             );
+            if (__kmp_generate_warnings == kmp_warnings_off) {
+                __kmp_str_free(&err_code.str);
+            }
         }; // if
     #endif
 
@@ -1274,6 +1163,7 @@ __kmp_create_monitor( kmp_info_t *th )
     KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
 
 } // __kmp_create_monitor
+#endif // KMP_USE_MONITOR
 
 void
 __kmp_exit_thread(
@@ -1282,6 +1172,7 @@ __kmp_exit_thread(
     pthread_exit( (void *)(intptr_t) exit_status );
 } // __kmp_exit_thread
 
+#if KMP_USE_MONITOR
 void __kmp_resume_monitor();
 
 void
@@ -1305,27 +1196,23 @@ __kmp_reap_monitor( kmp_info_t *th )
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 
 
-    /* First, check to see whether the monitor thread exists.  This could prevent a hang,
-       but if the monitor dies after the pthread_kill call and before the pthread_join
-       call, it will still hang. */
+    /* First, check to see whether the monitor thread exists to wake it up. This is
+       to avoid performance problem when the monitor sleeps during blocktime-size
+       interval */
 
     status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
-    if (status == ESRCH) {
-
-        KA_TRACE( 10, ("__kmp_reap_monitor: monitor does not exist, returning\n") );
-
-    } else
-    {
+    if (status != ESRCH) {
         __kmp_resume_monitor();   // Wake up the monitor thread
-        status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
-        if (exit_val != th) {
-            __kmp_msg(
-                kmp_ms_fatal,
-                KMP_MSG( ReapMonitorError ),
-                KMP_ERR( status ),
-                __kmp_msg_null
-            );
-        }
+    }
+    KA_TRACE( 10, ("__kmp_reap_monitor: try to join with monitor\n") );
+    status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
+    if (exit_val != th) {
+        __kmp_msg(
+            kmp_ms_fatal,
+            KMP_MSG( ReapMonitorError ),
+            KMP_ERR( status ),
+            __kmp_msg_null
+        );
     }
 
     th->th.th_info.ds.ds_tid  = KMP_GTID_DNE;
@@ -1337,6 +1224,7 @@ __kmp_reap_monitor( kmp_info_t *th )
     KMP_MB();       /* Flush all pending memory write invalidates.  */
 
 }
+#endif // KMP_USE_MONITOR
 
 void
 __kmp_reap_worker( kmp_info_t *th )
@@ -1348,39 +1236,17 @@ __kmp_reap_worker( kmp_info_t *th )
 
     KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
 
-    /* First, check to see whether the worker thread exists.  This could prevent a hang,
-       but if the worker dies after the pthread_kill call and before the pthread_join
-       call, it will still hang. */
-
-        {
-            status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
-            if (status == ESRCH) {
-                KA_TRACE( 10, ("__kmp_reap_worker: worker T#%d does not exist, returning\n",
-                               th->th.th_info.ds.ds_gtid ) );
-            }
-            else {
-                KA_TRACE( 10, ("__kmp_reap_worker: try to join with worker T#%d\n",
-                               th->th.th_info.ds.ds_gtid ) );
-
-                status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
+    status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
 #ifdef KMP_DEBUG
-                /* Don't expose these to the user until we understand when they trigger */
-                if ( status != 0 ) {
-                    __kmp_msg(
-                              kmp_ms_fatal,
-                              KMP_MSG( ReapWorkerError ),
-                              KMP_ERR( status ),
-                              __kmp_msg_null
-                              );
-                }
-                if ( exit_val != th ) {
-                    KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, "
-                                    "exit_val = %p\n",
-                                    th->th.th_info.ds.ds_gtid, exit_val ) );
-                }
+    /* Don't expose these to the user until we understand when they trigger */
+    if ( status != 0 ) {
+        __kmp_msg(kmp_ms_fatal, KMP_MSG( ReapWorkerError ), KMP_ERR( status ), __kmp_msg_null);
+    }
+    if ( exit_val != th ) {
+        KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, exit_val = %p\n",
+                        th->th.th_info.ds.ds_gtid, exit_val ) );
+    }
 #endif /* KMP_DEBUG */
-            }
-        }
 
     KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
 
@@ -1587,7 +1453,9 @@ __kmp_atfork_child (void)
     ++__kmp_fork_count;
 
     __kmp_init_runtime = FALSE;
+#if KMP_USE_MONITOR
     __kmp_init_monitor = 0;
+#endif
     __kmp_init_parallel = FALSE;
     __kmp_init_middle = FALSE;
     __kmp_init_serial = FALSE;
@@ -1635,7 +1503,7 @@ __kmp_atfork_child (void)
        Suggestion is to make the library initialization lazier, similar
        to what done for __kmpc_begin(). */
     // TODO: synchronize all static initializations with regular library
-    //       startup; look at kmp_global.c and etc.
+    //       startup; look at kmp_global.cpp and etc.
     //__kmp_internal_begin ();
 
 }
@@ -1662,6 +1530,7 @@ __kmp_suspend_initialize( void )
 static void
 __kmp_suspend_initialize_thread( kmp_info_t *th )
 {
+    ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count);
     if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
         /* this means we haven't initialized the suspension pthread objects for this thread
            in this instance of the process */
@@ -1671,6 +1540,7 @@ __kmp_suspend_initialize_thread( kmp_info_t *th )
         status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
         KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
         *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
+        ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count);
     };
 }
 
@@ -1701,7 +1571,7 @@ __kmp_suspend_uninitialize_thread( kmp_info_t *th )
 template <class C>
 static inline void __kmp_suspend_template( int th_gtid, C *flag )
 {
-    KMP_TIME_DEVELOPER_BLOCK(USER_suspend);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
     kmp_info_t *th = __kmp_threads[th_gtid];
     int status;
     typename C::flag_t old_spin;
@@ -1752,8 +1622,6 @@ static inline void __kmp_suspend_template( int th_gtid, C *flag )
                     KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
                 }
                 deactivated = TRUE;
-
-
             }
 
 #if USE_SUSPEND_TIMEOUT
@@ -1813,7 +1681,6 @@ static inline void __kmp_suspend_template( int th_gtid, C *flag )
     }
 #endif
 
-
     status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
     KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
 
@@ -1838,7 +1705,7 @@ void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
 template <class C>
 static inline void __kmp_resume_template( int target_gtid, C *flag )
 {
-    KMP_TIME_DEVELOPER_BLOCK(USER_resume);
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
     kmp_info_t *th = __kmp_threads[target_gtid];
     int status;
 
@@ -1872,7 +1739,6 @@ static inline void __kmp_resume_template( int target_gtid, C *flag )
             KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
                            "%u => %u\n",
                            gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
-
             status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
             KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
             return;
@@ -1892,7 +1758,6 @@ static inline void __kmp_resume_template( int target_gtid, C *flag )
     }
 #endif
 
-
     status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
     KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
     status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
@@ -1911,9 +1776,11 @@ void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
     __kmp_resume_template(target_gtid, flag);
 }
 
+#if KMP_USE_MONITOR
 void
 __kmp_resume_monitor()
 {
+    KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
     int status;
 #ifdef KMP_DEBUG
     int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
@@ -1937,6 +1804,7 @@ __kmp_resume_monitor()
     KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
                     gtid, KMP_GTID_MONITOR ) );
 }
+#endif // KMP_USE_MONITOR
 
 /* ------------------------------------------------------------------------ */
 /* ------------------------------------------------------------------------ */
@@ -1944,7 +1812,11 @@ __kmp_resume_monitor()
 void
 __kmp_yield( int cond )
 {
-    if (cond && __kmp_yielding_on) {
+    if (cond
+#if KMP_USE_MONITOR
+        && __kmp_yielding_on
+#endif
+    ) {
         sched_yield();
     }
 }
@@ -1955,18 +1827,21 @@ __kmp_yield( int cond )
 void
 __kmp_gtid_set_specific( int gtid )
 {
-    int status;
-    KMP_ASSERT( __kmp_init_runtime );
-    status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
-    KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
+    if( __kmp_init_gtid ) {
+        int status;
+        status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
+        KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
+    } else {
+        KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
+    }
 }
 
 int
 __kmp_gtid_get_specific()
 {
     int gtid;
-    if ( !__kmp_init_runtime ) {
-        KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
+    if ( !__kmp_init_gtid ) {
+        KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
         return KMP_GTID_SHUTDOWN;
     }
     gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
@@ -2269,6 +2144,29 @@ __kmp_elapsed_tick( double *t )
     *t = 1 / (double) CLOCKS_PER_SEC;
 }
 
+/* Return the current time stamp in nsec */
+kmp_uint64
+__kmp_now_nsec()
+{
+    struct timeval t;
+    gettimeofday(&t, NULL);
+    return KMP_NSEC_PER_SEC*t.tv_sec + 1000*t.tv_usec;
+}
+
+#if KMP_ARCH_X86 || KMP_ARCH_X86_64
+/* Measure clock tick per nanosecond */
+void
+__kmp_initialize_system_tick()
+{
+    kmp_uint64 delay = 100000; // 50~100 usec on most machines.
+    kmp_uint64 nsec = __kmp_now_nsec();
+    kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
+    kmp_uint64 now;
+    while ((now = __kmp_hardware_timestamp()) < goal);
+    __kmp_ticks_per_nsec = 1.0 * (delay + (now - goal)) / (__kmp_now_nsec() - nsec);
+}
+#endif
+
 /*
     Determine whether the given address is mapped into the current address space.
 */
@@ -2615,18 +2513,18 @@ __kmp_get_load_balance( int max )
 
 #endif // USE_LOAD_BALANCE
 
-#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC)
+#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || (KMP_OS_LINUX && KMP_ARCH_AARCH64) || KMP_ARCH_PPC64)
 
 // we really only need the case with 1 argument, because CLANG always build
 // a struct of pointers to shared variables referenced in the outlined function
 int
 __kmp_invoke_microtask( microtask_t pkfn,
                         int gtid, int tid,
-                        int argc, void *p_argv[] 
+                        int argc, void *p_argv[]
 #if OMPT_SUPPORT
                         , void **exit_frame_ptr
 #endif
-) 
+)
 {
 #if OMPT_SUPPORT
   *exit_frame_ptr = __builtin_frame_address(0);