Merging r361158:

------------------------------------------------------------------------
r361158 | jprotze | 2019-05-20 07:21:42 -0700 (Mon, 20 May 2019) | 11 lines

[OpenMP][OMPT] Fix locking testcases for 32 bit architectures

https://reviews.llvm.org/D58454 did not fix the problem for a typical use
case of building LLVM with gcc or icc and then testing with the newly built
clang compiler.
The compilers do not agree on how to extend a 32-bit pointer to uint64, so
make the pointer unsigned first, before adjusting the size.

Patch by Joachim Protze

Differential Revision: https://reviews.llvm.org/D58506
------------------------------------------------------------------------

git-svn-id: https://llvm.org/svn/llvm-project/openmp/branches/release_80@363628 91177308-0d34-0410-b5e6-96231b3b80d8

CMakeLists.txt

@@ -18,6 +18,8 @@ if (OPENMP_STANDALONE_BUILD OR "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_S
     "Enable -Werror flags to turn warnings into errors for supporting compilers.")
   set(OPENMP_LIBDIR_SUFFIX "" CACHE STRING
     "Suffix of lib installation directory, e.g. 64 => lib64")
+  # Do not use OPENMP_LIBDIR_SUFFIX directly, use OPENMP_INSTALL_LIBDIR.
+  set(OPENMP_INSTALL_LIBDIR "lib${OPENMP_LIBDIR_SUFFIX}")
 
   # Group test settings.
   set(OPENMP_TEST_C_COMPILER ${CMAKE_C_COMPILER} CACHE STRING
@@ -28,7 +30,7 @@ if (OPENMP_STANDALONE_BUILD OR "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_S
 else()
   set(OPENMP_ENABLE_WERROR ${LLVM_ENABLE_WERROR})
   # If building in tree, we honor the same install suffix LLVM uses.
-  set(OPENMP_LIBDIR_SUFFIX ${LLVM_LIBDIR_SUFFIX})
+  set(OPENMP_INSTALL_LIBDIR "lib${LLVM_LIBDIR_SUFFIX}")
 
   if (NOT MSVC)
     set(OPENMP_TEST_C_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)

LICENSE.txt

@@ -14,7 +14,7 @@ software contained in this directory tree is included below.
 University of Illinois/NCSA
 Open Source License
 
-Copyright (c) 1997-2016 Intel Corporation
+Copyright (c) 1997-2019 Intel Corporation
 
 All rights reserved.
 
@@ -51,7 +51,7 @@ SOFTWARE.
 
 ==============================================================================
 
-Copyright (c) 1997-2016 Intel Corporation
+Copyright (c) 1997-2019 Intel Corporation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -166,7 +166,7 @@ Options for ``libomp``
   Create the Fortran modules (requires Fortran compiler).
 
 macOS* Fat Libraries
-""""""""""""""""""
+""""""""""""""""""""
 On macOS* machines, it is possible to build universal (or fat) libraries which
 include both i386 and x86_64 architecture objects in a single archive.
 
@@ -194,7 +194,7 @@ Optional Features
 **LIBOMP_OMPT_SUPPORT** = ``ON|OFF``
   Include support for the OpenMP Tools Interface (OMPT). 
   This option is supported and ``ON`` by default for x86, x86_64, AArch64, and 
-  PPC64 on Linux, Windows, and mac OS.
+  PPC64 on Linux* and macOS*.
   This option is ``OFF`` if this feature is not supported for the platform.
 
 **LIBOMP_OMPT_OPTIONAL** = ``ON|OFF``
@@ -254,6 +254,42 @@ Options for ``libomptarget``
   Path of the folder that contains ``libomp.so``.  This is required for testing
   out-of-tree builds.
 
+Options for ``NVPTX device RTL``
+--------------------------------
+
+**LIBOMPTARGET_NVPTX_ENABLE_BCLIB** = ``ON|OFF``
+  Enable CUDA LLVM bitcode offloading device RTL. This is used for link time
+  optimization of the OMP runtime and application code. This option is enabled
+  by default if the build system determines that `CMAKE_C_COMPILER` is able to
+  compile and link the library.
+
+**LIBOMPTARGET_NVPTX_CUDA_COMPILER** = ``""``
+  Location of a CUDA compiler capable of emitting LLVM bitcode. Currently only
+  the Clang compiler is supported. This is only used when building the CUDA LLVM
+  bitcode offloading device RTL. If unspecified and the CMake C compiler is
+  Clang, then Clang is used.
+
+**LIBOMPTARGET_NVPTX_BC_LINKER** = ``""``
+  Location of a linker capable of linking LLVM bitcode objects. This is only
+  used when building the CUDA LLVM bitcode offloading device RTL. If unspecified
+  and the CMake C compiler is Clang and there exists a llvm-link binary in the
+  directory containing Clang, then this llvm-link binary is used.
+
+**LIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER** = ``""``
+  Host compiler to use with NVCC. This compiler is not going to be used to
+  produce any binary. Instead, this is used to overcome the input compiler
+  checks done by NVCC. E.g. if using a default host compiler that is not
+  compatible with NVCC, this option can be use to pass to NVCC a valid compiler
+  to avoid the error.
+
+ **LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES** = ``35``
+  List of CUDA compute capabilities that should be supported by the NVPTX
+  device RTL. E.g. for compute capabilities 6.0 and 7.0, the option "60,70"
+  should be used. Compute capability 3.5 is the minimum required.
+
+ **LIBOMPTARGET_NVPTX_DEBUG** = ``OFF|ON``
+  Enable printing of debug messages from the NVPTX device RTL.
+
 Example Usages of CMake
 =======================
 

cmake/DetectTestCompiler/CMakeLists.txt

@@ -1,11 +1,14 @@
 cmake_minimum_required(VERSION 2.8)
 project(DetectTestCompiler C CXX)
 
+include(CheckCCompilerFlag)
+include(CheckCXXCompilerFlag)
+
 function(write_compiler_information lang)
   set(information "${CMAKE_${lang}_COMPILER}")
   set(information "${information}\\;${CMAKE_${lang}_COMPILER_ID}")
   set(information "${information}\\;${CMAKE_${lang}_COMPILER_VERSION}")
-  set(information "${information}\\;${OpenMP_${lang}_FLAGS}")
+  set(information "${information}\\;${${lang}_FLAGS}")
   file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${lang}CompilerInformation.txt ${information})
 endfunction(write_compiler_information)
 
@@ -15,5 +18,22 @@ if (NOT OpenMP_Found)
   set(OpenMP_CXX_FLAGS "-fopenmp")
 endif()
 
+set(C_FLAGS ${flags} ${OpenMP_C_FLAGS})
+set(CXX_FLAGS ${flags} ${OpenMP_CXX_FLAGS})
+
+# TODO: Implement blockaddress in GlobalISel and remove this flag!
+if (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+  check_c_compiler_flag("-fno-experimental-isel" C_HAS_EXPERIMENTAL_ISEL_FLAG)
+  check_cxx_compiler_flag("-fno-experimental-isel" CXX_HAS_EXPERIMENTAL_ISEL_FLAG)
+  macro(add_experimental_isel_flag lang)
+    if (${lang}_HAS_EXPERIMENTAL_ISEL_FLAG)
+      set(${lang}_FLAGS "-fno-experimental-isel ${${lang}_FLAGS}")
+    endif()
+  endmacro(add_experimental_isel_flag)
+
+  add_experimental_isel_flag(C)
+  add_experimental_isel_flag(CXX)
+endif()
+
 write_compiler_information(C)
 write_compiler_information(CXX)

cmake/HandleOpenMPOptions.cmake

@@ -13,4 +13,7 @@ if (${OPENMP_ENABLE_WERROR})
   append_if(OPENMP_HAVE_WERROR_FLAG "-Werror" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 endif()
 
-append_if(OPENMP_HAVE_STD_CPP11_FLAG "-std=c++11" CMAKE_CXX_FLAGS)
+append_if(OPENMP_HAVE_STD_GNUPP11_FLAG "-std=gnu++11" CMAKE_CXX_FLAGS)
+if (NOT OPENMP_HAVE_STD_GNUPP11_FLAG)
+  append_if(OPENMP_HAVE_STD_CPP11_FLAG "-std=c++11" CMAKE_CXX_FLAGS)
+endif()

cmake/OpenMPTesting.cmake

@@ -87,7 +87,9 @@ function(set_test_compiler_information dir)
 
     # Determine major version.
     string(REGEX MATCH "[0-9]+" major "${OPENMP_TEST_C_COMPILER_VERSION}")
+    string(REGEX MATCH "[0-9]+\\.[0-9]+" majorminor "${OPENMP_TEST_C_COMPILER_VERSION}")
     set(OPENMP_TEST_COMPILER_VERSION_MAJOR "${major}" PARENT_SCOPE)
+    set(OPENMP_TEST_COMPILER_VERSION_MAJOR_MINOR "${majorminor}" PARENT_SCOPE)
   endif()
 endfunction()
 
@@ -97,7 +99,7 @@ if (${OPENMP_STANDALONE_BUILD})
   # project is built which is too late for detecting the compiler...
   file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DetectTestCompiler)
   execute_process(
-    COMMAND ${CMAKE_COMMAND} ${CMAKE_CURRENT_LIST_DIR}/DetectTestCompiler
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${CMAKE_CURRENT_LIST_DIR}/DetectTestCompiler
       -DCMAKE_C_COMPILER=${OPENMP_TEST_C_COMPILER}
       -DCMAKE_CXX_COMPILER=${OPENMP_TEST_CXX_COMPILER}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/DetectTestCompiler
@@ -117,7 +119,9 @@ else()
   # Cannot use CLANG_VERSION because we are not guaranteed that this is already set.
   set(OPENMP_TEST_COMPILER_VERSION "${LLVM_VERSION}")
   set(OPENMP_TEST_COMPILER_VERSION_MAJOR "${LLVM_MAJOR_VERSION}")
-  set(OPENMP_TEST_COMPILER_OPENMP_FLAGS "-fopenmp")
+  set(OPENMP_TEST_COMPILER_VERSION_MAJOR_MINOR "${LLVM_MAJOR_VERSION}.${LLVM_MINOR_VERSION}")
+  # TODO: Implement blockaddress in GlobalISel and remove this flag!
+  set(OPENMP_TEST_COMPILER_OPENMP_FLAGS "-fopenmp -fno-experimental-isel")
 endif()
 
 # Function to set compiler features for use in lit.
@@ -130,7 +134,7 @@ function(set_test_compiler_features)
     # Just use the lowercase of the compiler ID as fallback.
     string(TOLOWER "${OPENMP_TEST_COMPILER_ID}" comp)
   endif()
-  set(OPENMP_TEST_COMPILER_FEATURES "['${comp}', '${comp}-${OPENMP_TEST_COMPILER_VERSION_MAJOR}', '${comp}-${OPENMP_TEST_COMPILER_VERSION}']" PARENT_SCOPE)
+  set(OPENMP_TEST_COMPILER_FEATURES "['${comp}', '${comp}-${OPENMP_TEST_COMPILER_VERSION_MAJOR}', '${comp}-${OPENMP_TEST_COMPILER_VERSION_MAJOR_MINOR}', '${comp}-${OPENMP_TEST_COMPILER_VERSION}']" PARENT_SCOPE)
 endfunction()
 set_test_compiler_features()
 
@@ -143,7 +147,7 @@ function(add_openmp_testsuite target comment)
     return()
   endif()
 
-  cmake_parse_arguments(ARG "" "" "DEPENDS" ${ARGN})
+  cmake_parse_arguments(ARG "" "" "DEPENDS;ARGS" ${ARGN})
   # EXCLUDE_FROM_ALL excludes the test ${target} out of check-openmp.
   if (NOT EXCLUDE_FROM_ALL)
     # Register the testsuites and depends for the check-openmp rule.
@@ -152,8 +156,9 @@ function(add_openmp_testsuite target comment)
   endif()
 
   if (${OPENMP_STANDALONE_BUILD})
+    set(LIT_ARGS ${OPENMP_LIT_ARGS} ${ARG_ARGS})
     add_custom_target(${target}
-      COMMAND ${PYTHON_EXECUTABLE} ${OPENMP_LLVM_LIT_EXECUTABLE} ${OPENMP_LIT_ARGS} ${ARG_UNPARSED_ARGUMENTS}
+      COMMAND ${PYTHON_EXECUTABLE} ${OPENMP_LLVM_LIT_EXECUTABLE} ${LIT_ARGS} ${ARG_UNPARSED_ARGUMENTS}
       COMMENT ${comment}
       DEPENDS ${ARG_DEPENDS}
       ${cmake_3_2_USES_TERMINAL}
@@ -163,6 +168,7 @@ function(add_openmp_testsuite target comment)
       ${comment}
       ${ARG_UNPARSED_ARGUMENTS}
       DEPENDS clang clang-headers FileCheck ${ARG_DEPENDS}
+      ARGS ${ARG_ARGS}
     )
   endif()
 endfunction()

cmake/config-ix.cmake

@@ -3,4 +3,5 @@ include(CheckCXXCompilerFlag)
 
 check_c_compiler_flag(-Werror OPENMP_HAVE_WERROR_FLAG)
 
-check_cxx_compiler_flag(-std=c++11 OPENMP_HAVE_STD_CPP11_FLAG)
+check_cxx_compiler_flag(-std=gnu++11 OPENMP_HAVE_STD_GNUPP11_FLAG)
+check_cxx_compiler_flag(-std=c++11 OPENMP_HAVE_STD_CPP11_FLAG)

libomptarget/CMakeLists.txt

@@ -20,6 +20,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules ${CMAKE_MODULE_P
 
 if(OPENMP_STANDALONE_BUILD)
   # Build all libraries into a common place so that tests can find them.
+  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
   set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 endif()
 
@@ -40,10 +41,6 @@ set (LIBOMPTARGET_ALL_TARGETS "${LIBOMPTARGET_ALL_TARGETS} nvptx64-nvidia-cuda")
 # the list of supported targets in the current system.
 set (LIBOMPTARGET_SYSTEM_TARGETS "")
 
-# Set base directories - required for lit to locate the tests.
-set(LIBOMPTARGET_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-set(LIBOMPTARGET_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
-
 # If building this library in debug mode, we define a macro to enable
 # dumping progress messages at runtime.
 string( TOLOWER "${CMAKE_BUILD_TYPE}" LIBOMPTARGET_CMAKE_BUILD_TYPE)
@@ -65,8 +62,20 @@ if(NOT LIBOMPTARGET_LIBRARY_DIR)
   set(LIBOMPTARGET_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 endif()
 
+# Definitions for testing, for reuse when testing libomptarget-nvptx.
+if(OPENMP_STANDALONE_BUILD)
+  set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/../runtime/src" CACHE STRING
+    "Path to folder containing omp.h")
+  set(LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/../runtime/src" CACHE STRING
+    "Path to folder containing libomp.so")
+else()
+  set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/../runtime/src")
+endif()
+
+
 # Build offloading plugins and device RTLs if they are available.
 add_subdirectory(plugins)
+add_subdirectory(deviceRTLs)
 
 # Add tests.
 add_subdirectory(test)

libomptarget/cmake/Modules/LibomptargetGetDependencies.cmake

@@ -112,13 +112,82 @@ mark_as_advanced(
 ################################################################################
 # Looking for CUDA...
 ################################################################################
+if (CUDA_TOOLKIT_ROOT_DIR)
+  set(LIBOMPTARGET_CUDA_TOOLKIT_ROOT_DIR_PRESET TRUE)
+endif()
 find_package(CUDA QUIET)
 
 set(LIBOMPTARGET_DEP_CUDA_FOUND ${CUDA_FOUND})
-set(LIBOMPTARGET_DEP_CUDA_LIBRARIES ${CUDA_LIBRARIES})
 set(LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS})
 
 mark_as_advanced(
   LIBOMPTARGET_DEP_CUDA_FOUND 
-  LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS
-  LIBOMPTARGET_DEP_CUDA_LIBRARIES)
+  LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS)
+
+################################################################################
+# Looking for CUDA Driver API... (needed for CUDA plugin)
+################################################################################
+
+find_library (
+    LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES
+  NAMES
+    cuda
+  PATHS
+    /lib64)
+
+# There is a libcuda.so in lib64/stubs that can be used for linking.
+if (NOT LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES AND CUDA_FOUND)
+  # Since CMake 3.3 FindCUDA.cmake defaults to using static libraries. In this
+  # case CUDA_LIBRARIES contains additional linker arguments which breaks
+  # get_filename_component below. Fortunately, since that change the module
+  # exports CUDA_cudart_static_LIBRARY which points to a single file in the
+  # right directory.
+  set(cuda_library ${CUDA_LIBRARIES})
+  if (DEFINED CUDA_cudart_static_LIBRARY)
+    set(cuda_library ${CUDA_cudart_static_LIBRARY})
+  endif()
+  get_filename_component(CUDA_LIBDIR ${cuda_library} DIRECTORY)
+  find_library (
+      LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES
+    NAMES
+      cuda
+    HINTS
+      "${CUDA_LIBDIR}/stubs")
+endif()
+
+find_package_handle_standard_args(
+  LIBOMPTARGET_DEP_CUDA_DRIVER
+  DEFAULT_MSG
+  LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES)
+
+mark_as_advanced(LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES)
+
+################################################################################
+# Looking for CUDA libdevice subdirectory
+#
+# Special case for Debian/Ubuntu to have nvidia-cuda-toolkit work
+# out of the box. More info on http://bugs.debian.org/882505
+################################################################################
+
+set(LIBOMPTARGET_CUDA_LIBDEVICE_SUBDIR nvvm/libdevice)
+
+# Don't alter CUDA_TOOLKIT_ROOT_DIR if the user specified it, if a value was
+# already cached for it, or if it already has libdevice.  Otherwise, on
+# Debian/Ubuntu, look where the nvidia-cuda-toolkit package normally installs
+# libdevice.
+if (NOT LIBOMPTARGET_CUDA_TOOLKIT_ROOT_DIR_PRESET AND
+    NOT EXISTS
+      "${CUDA_TOOLKIT_ROOT_DIR}/${LIBOMPTARGET_CUDA_LIBDEVICE_SUBDIR}")
+  find_program(LSB_RELEASE lsb_release)
+  if (LSB_RELEASE)
+    execute_process(COMMAND ${LSB_RELEASE} -is
+      OUTPUT_VARIABLE LSB_RELEASE_ID
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    set(candidate_dir /usr/lib/cuda)
+    if ((LSB_RELEASE_ID STREQUAL "Debian" OR LSB_RELEASE_ID STREQUAL "Ubuntu")
+        AND EXISTS "${candidate_dir}/${LIBOMPTARGET_CUDA_LIBDEVICE_SUBDIR}")
+      set(CUDA_TOOLKIT_ROOT_DIR "${candidate_dir}" CACHE PATH
+          "Toolkit location." FORCE)
+    endif()
+  endif()
+endif()

libomptarget/cmake/Modules/LibomptargetNVPTXBitcodeLibrary.cmake

@@ -0,0 +1,112 @@
+#
+#//===----------------------------------------------------------------------===//
+#//
+#//                     The LLVM Compiler Infrastructure
+#//
+#// This file is dual licensed under the MIT and the University of Illinois Open
+#// Source Licenses. See LICENSE.txt for details.
+#//
+#//===----------------------------------------------------------------------===//
+#
+
+# We use the compiler and linker provided by the user, attempt to use the one
+# used to build libomptarget or just fail.
+set(LIBOMPTARGET_NVPTX_BCLIB_SUPPORTED FALSE)
+
+if (NOT LIBOMPTARGET_NVPTX_CUDA_COMPILER STREQUAL "")
+  set(LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER ${LIBOMPTARGET_NVPTX_CUDA_COMPILER})
+elseif(${CMAKE_C_COMPILER_ID} STREQUAL "Clang")
+  set(LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER ${CMAKE_C_COMPILER})
+else()
+  return()
+endif()
+
+# Get compiler directory to try to locate a suitable linker.
+get_filename_component(compiler_dir ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER} DIRECTORY)
+set(llvm_link "${compiler_dir}/llvm-link")
+
+if (NOT LIBOMPTARGET_NVPTX_BC_LINKER STREQUAL "")
+  set(LIBOMPTARGET_NVPTX_SELECTED_BC_LINKER ${LIBOMPTARGET_NVPTX_BC_LINKER})
+elseif (EXISTS "${llvm_link}")
+  # Use llvm-link from the compiler directory.
+  set(LIBOMPTARGET_NVPTX_SELECTED_BC_LINKER "${llvm_link}")
+else()
+  return()
+endif()
+
+function(try_compile_bitcode output source)
+  set(srcfile ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/src.cu)
+  file(WRITE ${srcfile} "${source}\n")
+  set(bcfile ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/out.bc)
+
+  # The remaining arguments are the flags to be tested.
+  # FIXME: Don't hardcode GPU version. This is currently required because
+  #        Clang refuses to compile its default of sm_20 with CUDA 9.
+  execute_process(
+    COMMAND ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER} ${ARGN}
+      --cuda-gpu-arch=sm_35 -c ${srcfile} -o ${bcfile}
+    RESULT_VARIABLE result
+    OUTPUT_QUIET ERROR_QUIET)
+  if (result EQUAL 0)
+    set(${output} TRUE PARENT_SCOPE)
+  else()
+    set(${output} FALSE PARENT_SCOPE)
+  endif()
+endfunction()
+
+# Save for which compiler we are going to do the following checks so that we
+# can discard cached values if the user specifies a different value.
+set(discard_cached FALSE)
+if (DEFINED LIBOMPTARGET_NVPTX_CHECKED_CUDA_COMPILER AND
+    NOT("${LIBOMPTARGET_NVPTX_CHECKED_CUDA_COMPILER}" STREQUAL "${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER}"))
+  set(discard_cached TRUE)
+endif()
+set(LIBOMPTARGET_NVPTX_CHECKED_CUDA_COMPILER "${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER}" CACHE INTERNAL "" FORCE)
+
+function(check_bitcode_compilation output source)
+  if (${discard_cached} OR NOT DEFINED ${output})
+    message(STATUS "Performing Test ${output}")
+    # Forward additional arguments which contain the flags.
+    try_compile_bitcode(result "${source}" ${ARGN})
+    set(${output} ${result} CACHE INTERNAL "" FORCE)
+    if(${result})
+      message(STATUS "Performing Test ${output} - Success")
+    else()
+      message(STATUS "Performing Test ${output} - Failed")
+    endif()
+  endif()
+endfunction()
+
+# These flags are required to emit LLVM Bitcode. We check them together because
+# if any of them are not supported, there is no point in finding out which are.
+set(compiler_flags_required -emit-llvm -O1 --cuda-device-only --cuda-path=${CUDA_TOOLKIT_ROOT_DIR})
+set(compiler_flags_required_src "extern \"C\" __device__ int thread() { return threadIdx.x; }")
+check_bitcode_compilation(LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_FLAGS_REQUIRED "${compiler_flags_required_src}" ${compiler_flags_required})
+
+# It makes no sense to continue given that the compiler doesn't support
+# emitting basic LLVM Bitcode
+if (NOT LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_FLAGS_REQUIRED)
+  return()
+endif()
+
+set(LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER_FLAGS ${compiler_flags_required})
+
+# Declaring external shared device variables might need an additional flag
+# since Clang 7.0 and was entirely unsupported since version 4.0.
+set(extern_device_shared_src "extern __device__ __shared__ int test;")
+
+check_bitcode_compilation(LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_EXTERN_SHARED "${extern_device_shared_src}" ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER_FLAGS})
+if (NOT LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_EXTERN_SHARED)
+  set(compiler_flag_fcuda_rdc -fcuda-rdc)
+  set(compiler_flag_fcuda_rdc_full ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER_FLAGS} ${compiler_flag_fcuda_rdc})
+  check_bitcode_compilation(LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_FCUDA_RDC "${extern_device_shared_src}" ${compiler_flag_fcuda_rdc_full})
+
+  if (NOT LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_FCUDA_RDC)
+    return()
+  endif()
+
+  set(LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER_FLAGS "${compiler_flag_fcuda_rdc_full}")
+endif()
+
+# We can compile LLVM Bitcode from CUDA source code!
+set(LIBOMPTARGET_NVPTX_BCLIB_SUPPORTED TRUE)

libomptarget/deviceRTLs/CMakeLists.txt

@@ -0,0 +1,14 @@
+##===----------------------------------------------------------------------===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is dual licensed under the MIT and the University of Illinois Open
+# Source Licenses. See LICENSE.txt for details.
+#
+# ##===----------------------------------------------------------------------===##
+#
+# Build a device RTL for each available machine available.
+#
+##===----------------------------------------------------------------------===##
+
+add_subdirectory(nvptx)

libomptarget/deviceRTLs/nvptx/CMakeLists.txt

@@ -0,0 +1,186 @@
+##===----------------------------------------------------------------------===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is dual licensed under the MIT and the University of Illinois Open
+# Source Licenses. See LICENSE.txt for details.
+#
+##===----------------------------------------------------------------------===##
+#
+# Build the NVPTX (CUDA) Device RTL if the CUDA tools are available
+#
+##===----------------------------------------------------------------------===##
+
+set(LIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER "" CACHE STRING
+  "Path to alternate NVCC host compiler to be used by the NVPTX device RTL.")
+
+if(LIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER)
+  find_program(ALTERNATE_CUDA_HOST_COMPILER NAMES ${LIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER})
+  if(NOT ALTERNATE_CUDA_HOST_COMPILER)
+    libomptarget_say("Not building CUDA offloading device RTL: invalid NVPTX alternate host compiler.")
+  endif()
+  set(CUDA_HOST_COMPILER ${ALTERNATE_CUDA_HOST_COMPILER} CACHE FILEPATH "" FORCE)
+endif()
+
+# We can't use clang as nvcc host preprocessor, so we attempt to replace it with
+# gcc.
+if(CUDA_HOST_COMPILER MATCHES clang)
+
+  find_program(LIBOMPTARGET_NVPTX_ALTERNATE_GCC_HOST_COMPILER NAMES gcc)
+
+  if(NOT LIBOMPTARGET_NVPTX_ALTERNATE_GCC_HOST_COMPILER)
+    libomptarget_say("Not building CUDA offloading device RTL: clang is not supported as NVCC host compiler.")
+    libomptarget_say("Please include gcc in your path or set LIBOMPTARGET_NVPTX_ALTERNATE_HOST_COMPILER to the full path of of valid compiler.")
+    return()
+  endif()
+  set(CUDA_HOST_COMPILER "${LIBOMPTARGET_NVPTX_ALTERNATE_GCC_HOST_COMPILER}" CACHE FILEPATH "" FORCE)
+endif()
+
+if(LIBOMPTARGET_DEP_CUDA_FOUND)
+  libomptarget_say("Building CUDA offloading device RTL.")
+
+  # We really don't have any host code, so we don't need to care about
+  # propagating host flags.
+  set(CUDA_PROPAGATE_HOST_FLAGS OFF)
+
+  set(cuda_src_files
+      src/cancel.cu
+      src/critical.cu
+      src/data_sharing.cu
+      src/libcall.cu
+      src/loop.cu
+      src/omptarget-nvptx.cu
+      src/parallel.cu
+      src/reduction.cu
+      src/sync.cu
+      src/task.cu
+  )
+
+  set(omp_data_objects src/omp_data.cu)
+
+  # Get the compute capability the user requested or use SM_35 by default.
+  # SM_35 is what clang uses by default.
+  set(default_capabilities 35)
+  if (DEFINED LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITY)
+    set(default_capabilities ${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITY})
+    libomptarget_warning_say("LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITY is deprecated, please use LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES")
+  endif()
+  set(LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES ${default_capabilities} CACHE STRING
+    "List of CUDA Compute Capabilities to be used to compile the NVPTX device RTL.")
+  string(REPLACE "," ";" nvptx_sm_list ${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES})
+
+  foreach(sm ${nvptx_sm_list})
+    set(CUDA_ARCH ${CUDA_ARCH} -gencode arch=compute_${sm},code=sm_${sm})
+  endforeach()
+
+  # Activate RTL message dumps if requested by the user.
+  set(LIBOMPTARGET_NVPTX_DEBUG FALSE CACHE BOOL
+    "Activate NVPTX device RTL debug messages.")
+  if(${LIBOMPTARGET_NVPTX_DEBUG})
+    set(CUDA_DEBUG -DOMPTARGET_NVPTX_DEBUG=-1 -g --ptxas-options=-v)
+  endif()
+
+  # NVPTX runtime library has to be statically linked. Dynamic linking is not
+  # yet supported by the CUDA toolchain on the device.
+  set(BUILD_SHARED_LIBS OFF)
+  set(CUDA_SEPARABLE_COMPILATION ON)
+
+  cuda_add_library(omptarget-nvptx STATIC ${cuda_src_files} ${omp_data_objects}
+      OPTIONS ${CUDA_ARCH} ${CUDA_DEBUG})
+
+  # Install device RTL under the lib destination folder.
+  install(TARGETS omptarget-nvptx ARCHIVE DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+
+  target_link_libraries(omptarget-nvptx ${CUDA_LIBRARIES})
+
+
+  # Check if we can create an LLVM bitcode implementation of the runtime library
+  # that could be inlined in the user application. For that we need to find
+  # a Clang compiler capable of compiling our CUDA files to LLVM bitcode and
+  # an LLVM linker.
+  set(LIBOMPTARGET_NVPTX_CUDA_COMPILER "" CACHE STRING
+    "Location of a CUDA compiler capable of emitting LLVM bitcode.")
+  set(LIBOMPTARGET_NVPTX_BC_LINKER "" CACHE STRING
+    "Location of a linker capable of linking LLVM bitcode objects.")
+
+  include(LibomptargetNVPTXBitcodeLibrary)
+
+  set(bclib_default FALSE)
+  if (${LIBOMPTARGET_NVPTX_BCLIB_SUPPORTED})
+    set(bclib_default TRUE)
+  endif()
+  set(LIBOMPTARGET_NVPTX_ENABLE_BCLIB ${bclib_default} CACHE BOOL
+    "Enable CUDA LLVM bitcode offloading device RTL.")
+  if (${LIBOMPTARGET_NVPTX_ENABLE_BCLIB})
+    if (NOT ${LIBOMPTARGET_NVPTX_BCLIB_SUPPORTED})
+      libomptarget_error_say("Cannot build CUDA LLVM bitcode offloading device RTL!")
+    endif()
+    libomptarget_say("Building CUDA LLVM bitcode offloading device RTL.")
+
+    # Set flags for LLVM Bitcode compilation.
+    set(bc_flags ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER_FLAGS})
+    if(${LIBOMPTARGET_NVPTX_DEBUG})
+      set(bc_flags ${bc_flags} -DOMPTARGET_NVPTX_DEBUG=-1)
+    else()
+      set(bc_flags ${bc_flags} -DOMPTARGET_NVPTX_DEBUG=0)
+    endif()
+
+    # CUDA 9 header files use the nv_weak attribute which clang is not yet prepared
+    # to handle. Therefore, we use 'weak' instead. We are compiling only for the
+    # device, so it should be equivalent.
+    if(CUDA_VERSION_MAJOR GREATER 8)
+      set(bc_flags ${bc_flags} -Dnv_weak=weak)
+    endif()
+
+    # Create target to build all Bitcode libraries.
+    add_custom_target(omptarget-nvptx-bc)
+
+    # Generate a Bitcode library for all the compute capabilities the user requested.
+    foreach(sm ${nvptx_sm_list})
+      set(cuda_arch --cuda-gpu-arch=sm_${sm})
+
+      # Compile CUDA files to bitcode.
+      set(bc_files "")
+      foreach(src ${cuda_src_files})
+        get_filename_component(infile ${src} ABSOLUTE)
+        get_filename_component(outfile ${src} NAME)
+
+        add_custom_command(OUTPUT ${outfile}-sm_${sm}.bc
+          COMMAND ${LIBOMPTARGET_NVPTX_SELECTED_CUDA_COMPILER} ${bc_flags} ${cuda_arch}
+            -c ${infile} -o ${outfile}-sm_${sm}.bc
+          DEPENDS ${infile}
+          IMPLICIT_DEPENDS CXX ${infile}
+          COMMENT "Building LLVM bitcode ${outfile}-sm_${sm}.bc"
+          VERBATIM
+        )
+        set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${outfile}-sm_${sm}.bc)
+
+        list(APPEND bc_files ${outfile}-sm_${sm}.bc)
+      endforeach()
+
+      # Link to a bitcode library.
+      add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-nvptx-sm_${sm}.bc
+          COMMAND ${LIBOMPTARGET_NVPTX_SELECTED_BC_LINKER}
+            -o ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-nvptx-sm_${sm}.bc ${bc_files}
+          DEPENDS ${bc_files}
+          COMMENT "Linking LLVM bitcode libomptarget-nvptx-sm_${sm}.bc"
+      )
+      set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES libomptarget-nvptx-sm_${sm}.bc)
+
+      add_custom_target(omptarget-nvptx-${sm}-bc ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-nvptx-sm_${sm}.bc)
+      add_dependencies(omptarget-nvptx-bc omptarget-nvptx-${sm}-bc)
+
+      # Copy library to destination.
+      add_custom_command(TARGET omptarget-nvptx-${sm}-bc POST_BUILD
+                         COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-nvptx-sm_${sm}.bc
+                         $<TARGET_FILE_DIR:omptarget-nvptx>)
+
+      # Install bitcode library under the lib destination folder.
+      install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-nvptx-sm_${sm}.bc DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+    endforeach()
+  endif()
+
+  add_subdirectory(test)
+else()
+  libomptarget_say("Not building CUDA offloading device RTL: CUDA tools not found in the system.")
+endif()

libomptarget/deviceRTLs/nvptx/docs/ReductionDesign.txt

@@ -0,0 +1,523 @@
+
+**Design document for OpenMP reductions on the GPU** 
+
+//Abstract: //In this document we summarize the new design for an OpenMP
+implementation of reductions on NVIDIA GPUs.  This document comprises
+* a succinct background review,
+* an introduction to the decoupling of reduction algorithm and
+    data-structure-specific processing routines,
+* detailed illustrations of reduction algorithms used and
+* a brief overview of steps we have made beyond the last implementation.
+
+**Problem Review**
+
+Consider a typical OpenMP program with reduction pragma.
+
+```
+    double foo, bar;
+    #pragma omp parallel for reduction(+:foo, bar)
+    for (int i = 0; i < N; i++) {
+      foo+=A[i]; bar+=B[i];
+    }
+```
+where 'foo' and 'bar' are reduced across all threads in the parallel region.
+Our primary goal is to efficiently aggregate the values of foo and bar in
+such manner that
+* makes the compiler logically concise.
+* efficiently reduces within warps, threads, blocks and the device.
+
+**Introduction to Decoupling**
+In this section we address the problem of making the compiler
+//logically concise// by partitioning the task of reduction into two broad
+categories: data-structure specific routines and algorithmic routines.
+
+The previous reduction implementation was highly coupled with
+the specificity of the reduction element data structures (e.g., sizes, data
+types) and operators of the reduction (e.g., addition, multiplication). In
+our implementation we strive to decouple them. In our final implementations,
+we could remove all template functions in our runtime system.
+
+The (simplified) pseudo code generated by LLVM is as follows:
+
+```
+    1. Create private copies of variables: foo_p, bar_p
+    2. Each thread reduces the chunk of A and B assigned to it and writes
+       to foo_p and bar_p respectively.
+    3. ret = kmpc_nvptx_reduce_nowait(..., reduceData, shuffleReduceFn, 
+               interWarpCpyFn)
+        where:
+        struct ReduceData {
+          double *foo;
+          double *bar;
+        } reduceData
+        reduceData.foo = &foo_p
+        reduceData.bar = &bar_p
+
+        shuffleReduceFn and interWarpCpyFn are two auxiliary functions
+        generated to aid the runtime performing algorithmic steps
+        while being data-structure agnostic about ReduceData.
+
+        In particular, shuffleReduceFn is a function that takes the following
+        inputs:
+        a. local copy of ReduceData
+        b. its lane_id
+        c. the offset of the lane_id which hosts a remote ReduceData
+                relative to the current one
+        d. an algorithm version paramter determining which reduction
+                algorithm to use.
+        This shuffleReduceFn retrieves the remote ReduceData through shuffle
+        intrinsics and  reduces, using the algorithm specified by the 4th
+        parameter, the local ReduceData and with the remote ReduceData element
+        wise, and places the resultant values into the local ReduceData.
+
+        Different reduction algorithms are implemented with different runtime
+        functions, but they all make calls to this same shuffleReduceFn to
+        perform the essential reduction step. Therefore, based on the 4th
+        parameter, this shuffleReduceFn will behave slightly differently to
+        cooperate with the runtime function to ensure correctness under
+        different circumstances.
+
+        InterWarpCpyFn, as the name suggests, is a function that copies data
+        across warps. Its function is to tunnel all the thread private
+        ReduceData that is already reduced within a warp to a lane in the first
+        warp with minimal shared memory footprint. This is an essential step to
+        prepare for the last step of a block reduction.
+
+        (Warp, block, device level reduction routines that utilize these
+        auxiliary functions will be discussed in the next section.)
+
+    4. if ret == 1:
+        The master thread stores the reduced result in the globals.
+        foo += reduceData.foo; bar += reduceData.bar
+```
+
+**Reduction Algorithms**
+
+On the warp level, we have three versions of the algorithms:
+
+1. Full Warp Reduction
+
+```
+gpu_regular_warp_reduce(void *reduce_data,
+                        kmp_ShuffleReductFctPtr ShuffleReduceFn) {
+  for (int offset = WARPSIZE/2; offset > 0; offset /= 2)
+    ShuffleReduceFn(reduce_data, 0, offset, 0);
+}
+```
+ShuffleReduceFn is used here with lane_id set to 0 because it is not used
+therefore we save instructions by not retrieving lane_id from the corresponding
+special registers. The 4th parameters, which represents the version of the
+algorithm being used here, is set to 0 to signify full warp reduction.
+
+In this version specified (=0), the ShuffleReduceFn behaves, per element, as
+follows:
+
+```
+//reduce_elem refers to an element in the local ReduceData
+//remote_elem is retrieved from a remote lane
+remote_elem = shuffle_down(reduce_elem, offset, 32);
+reduce_elem = reduce_elem @ remote_elem;
+
+```
+
+An illustration of this algorithm operating on a hypothetical 8-lane full-warp
+would be:
+{F74}
+The coloring invariant follows that elements with the same color will be
+combined and reduced in the next reduction step. As can be observed, no overhead
+is present, exactly log(2, N) steps are needed.
+
+2. Contiguous Full Warp Reduction
+```
+gpu_irregular_warp_reduce(void *reduce_data,
+                          kmp_ShuffleReductFctPtr ShuffleReduceFn, int size,
+                          int lane_id) {
+  int curr_size;
+  int offset;
+    curr_size = size;
+    mask = curr_size/2;
+    while (offset>0) {
+      ShuffleReduceFn(reduce_data, lane_id, offset, 1);
+      curr_size = (curr_size+1)/2;
+      offset = curr_size/2;
+    }
+}
+```
+
+In this version specified (=1), the ShuffleReduceFn behaves, per element, as
+follows:
+```
+//reduce_elem refers to an element in the local ReduceData
+//remote_elem is retrieved from a remote lane
+remote_elem = shuffle_down(reduce_elem, offset, 32);
+if (lane_id < offset) {
+    reduce_elem = reduce_elem @ remote_elem
+} else {
+    reduce_elem = remote_elem
+}
+```
+
+An important invariant (also a restriction on the starting state of the
+reduction) is that this algorithm assumes that all unused ReduceData are
+located in a contiguous subset of threads in a warp starting from lane 0.
+
+With the presence of a trailing active lane with an odd-numbered lane
+id, its value will not be aggregated with any other lane. Therefore,
+in order to preserve the invariant, such ReduceData is copied to the first lane
+whose thread-local ReduceData has already being used in a previous reduction
+and would therefore be useless otherwise.
+
+An illustration of this algorithm operating on a hypothetical 8-lane partial
+warp woud be:
+{F75}
+
+As illustrated, this version of the algorithm introduces overhead whenever
+we have odd number of participating lanes in any reduction step to
+copy data between lanes.
+
+3. Dispersed Partial Warp Reduction
+```
+gpu_irregular_simt_reduce(void *reduce_data,
+                          kmp_ShuffleReductFctPtr ShuffleReduceFn) {
+  int size, remote_id;
+  int logical_lane_id = find_number_of_dispersed_active_lanes_before_me() * 2;
+  do {
+      remote_id = find_the_next_active_lane_id_right_after_me();
+      // the above function returns 0 of no active lane
+      // is present right after the current thread.
+      size = get_number_of_active_lanes_in_this_warp();
+      logical_lane_id /= 2;
+      ShuffleReduceFn(reduce_data, logical_lane_id, remote_id-1-threadIdx.x, 2);
+  } while (logical_lane_id % 2 == 0 && size > 1);
+```
+
+There is no assumption made about the initial state of the reduction.
+Any number of lanes (>=1) could be active at any position. The reduction
+result is kept in the first active lane.
+
+In this version specified (=2), the ShuffleReduceFn behaves, per element, as
+follows:
+```
+//reduce_elem refers to an element in the local ReduceData
+//remote_elem is retrieved from a remote lane
+remote_elem = shuffle_down(reduce_elem, offset, 32);
+if (LaneId % 2 == 0 && Offset > 0) {
+    reduce_elem = reduce_elem @ remote_elem
+} else {
+    reduce_elem = remote_elem
+}
+```
+We will proceed with a brief explanation for some arguments passed in,
+it is important to notice that, in this section, we will introduce the
+concept of logical_lane_id, and it is important to distinguish it
+from physical lane_id as defined by nvidia.
+1. //logical_lane_id//: as the name suggests, it refers to the calculated
+    lane_id (instead of the physical one defined by nvidia) that would make
+    our algorithm logically concise. A thread with logical_lane_id k means
+    there are (k-1) threads before it.
+2. //remote_id-1-threadIdx.x//: remote_id is indeed the nvidia-defined lane
+    id of the remote lane from which we will retrieve the ReduceData. We
+    subtract (threadIdx+1) from it because we would like to maintain only one
+    underlying shuffle intrinsic (which is used to communicate among lanes in a
+    warp). This particular version of shuffle intrinsic we take accepts only
+    offsets, instead of absolute lane_id. Therefore the subtraction is performed
+    on the absolute lane_id we calculated to obtain the offset.
+
+This algorithm is slightly different in 2 ways and it is not, conceptually, a
+generalization of the above algorithms.
+1. It reduces elements close to each other. For instance, values in the 0th lane
+    is to be combined with that of the 1st lane; values in the 2nd lane is to be
+    combined with that of the 3rd lane. We did not use the previous algorithm
+    where the first half of the (partial) warp is reduced with the second half
+    of the (partial) warp. This is because, the mapping
+    f(x): logical_lane_id -> physical_lane_id;
+    can be easily calculated whereas its inverse
+    f^-1(x): physical_lane_id -> logical_lane_id
+    cannot and performing such reduction requires the inverse to be known.
+2. Because this algorithm is agnostic about the positions of the lanes that are
+    active, we do not need to perform the coping step as in the second
+    algorithm.
+An illustrative run would look like
+{F76}
+As observed, overhead is high because in each and every step of reduction,
+logical_lane_id is recalculated; so is the remote_id.
+
+On a block level, we have implemented the following block reduce algorithm:
+
+```
+gpu_irregular_block_reduce(void *reduce_data,
+              kmp_ShuffleReductFctPtr shuflReduceFn,
+              kmp_InterWarpCopyFctPtr interWarpCpyFn,
+              int size) {
+
+  int wid = threadIdx.x/WARPSIZE;
+  int lane_id = threadIdx.x%WARPSIZE;
+
+  int warp_needed = (size+WARPSIZE-1)/WARPSIZE; //ceiling of division
+
+  unsigned tnum = __ballot(1);
+  int thread_num = __popc(tnum);
+
+    //full warp reduction
+    if (thread_num == WARPSIZE) {
+      gpu_regular_warp_reduce(reduce_data, shuflReduceFn);
+    }
+    //partial warp reduction
+    if (thread_num < WARPSIZE) {
+        gpu_irregular_warp_reduce(reduce_data, shuflReduceFn, thread_num,
+                                  lane_id);
+    }
+    //Gather all the reduced values from each warp
+    //to the first warp
+    //named_barrier inside this function to ensure
+    //correctness. It is effectively a sync_thread
+    //that won't deadlock.
+    interWarpCpyFn(reduce_data, warp_needed);
+
+    //This is to reduce data gathered from each "warp master".
+    if (wid==0) {
+        gpu_irregular_warp_reduce(reduce_data, shuflReduceFn, warp_needed,
+                                  lane_id);
+    }
+
+  return;
+}
+```
+In this function, no ShuffleReduceFn is directly called as it makes calls
+to various versions of the warp-reduction functions. It first reduces
+ReduceData warp by warp; in the end, we end up with the number of
+ReduceData equal to the number of warps present in this thread
+block. We then proceed to gather all such ReduceData to the first warp.
+
+As observed, in this algorithm we make use of the function InterWarpCpyFn,
+which copies data from each of the "warp master" (0th lane of each warp, where 
+a warp-reduced ReduceData is held) to the 0th warp. This step reduces (in a
+mathematical sense) the problem of reduction across warp masters in a block to
+the problem of warp reduction which we already have solutions to.
+
+We can thus completely avoid the use of atomics to reduce in a threadblock.
+
+**Efficient Cross Block Reduce**
+
+The next challenge is to reduce values across threadblocks.  We aim to do this
+without atomics or critical sections.
+
+Let a kernel be started with TB threadblocks.
+Let the GPU have S SMs.
+There can be at most N active threadblocks per SM at any time.
+
+Consider a threadblock tb (tb < TB) running on SM s (s < SM).  'tb' is one of
+at most 'N' active threadblocks on SM s.  Let each threadblock active on an SM
+be given an instance identifier id (0 <= id < N).  Therefore, the tuple (s, id)
+uniquely identifies an active threadblock on the GPU.
+
+To efficiently implement cross block reduce, we first allocate an array for
+each value to be reduced of size S*N (which is the maximum number of active
+threadblocks at any time on the device).
+
+Each threadblock reduces its value to slot [s][id].  This can be done without
+locking since no other threadblock can write to the same slot concurrently.
+
+As a final stage, we reduce the values in the array as follows:
+
+```
+// Compiler generated wrapper function for each target region with a reduction
+clause.
+target_function_wrapper(map_args, reduction_array)  <--- start with 1 team and 1
+   thread.
+  // Use dynamic parallelism to launch M teams, N threads as requested by the
+  user to execute the target region.
+
+  target_function<<M, N>>(map_args)
+
+  Reduce values in reduction_array
+
+```
+
+**Comparison with Last Version**
+
+
+The (simplified) pseudo code generated by LLVM on the host is as follows:
+
+
+```
+    1. Create private copies of variables: foo_p, bar_p
+    2. Each thread reduces the chunk of A and B assigned to it and writes
+       to foo_p and bar_p respectively.
+    3. ret = kmpc_reduce_nowait(..., reduceData, reduceFn, lock)
+        where:
+        struct ReduceData {
+          double *foo;
+          double *bar;
+        } reduceData
+        reduceData.foo = &foo_p
+        reduceData.bar = &bar_p
+
+        reduceFn is a pointer to a function that takes in two inputs
+        of type ReduceData, "reduces" them element wise, and places the
+        result in the first input:
+        reduceFn(ReduceData *a, ReduceData *b)
+          a = a @ b
+
+        Every thread in the parallel region calls kmpc_reduce_nowait with
+        its private copy of reduceData.  The runtime reduces across the
+        threads (using tree reduction on the operator 'reduceFn?) and stores
+        the final result in the master thread if successful.
+    4. if ret == 1:
+        The master thread stores the reduced result in the globals.
+        foo += reduceData.foo; bar += reduceData.bar
+    5. else if ret == 2:
+        In this case kmpc_reduce_nowait() could not use tree reduction,
+        so use atomics instead:
+        each thread atomically writes to foo
+        each thread atomically writes to bar
+```
+
+On a GPU, a similar reduction may need to be performed across SIMT threads,
+warps, and threadblocks.  The challenge is to do so efficiently in a fashion
+that is compatible with the LLVM OpenMP implementation.
+
+In the previously released 0.1 version of the LLVM OpenMP compiler for GPUs,
+the salient steps of the code generated are as follows:
+
+
+```
+    1. Create private copies of variables: foo_p, bar_p
+    2. Each thread reduces the chunk of A and B assigned to it and writes
+       to foo_p and bar_p respectively.
+    3. ret = kmpc_reduce_nowait(..., reduceData, reduceFn, lock)
+        status = can_block_reduce()
+        if status == 1:
+          reduce efficiently to thread 0 using shuffles and shared memory.
+          return 1
+        else
+          cannot use efficient block reduction, fallback to atomics
+          return 2
+    4. if ret == 1:
+        The master thread stores the reduced result in the globals.
+        foo += reduceData.foo; bar += reduceData.bar
+    5. else if ret == 2:
+        In this case kmpc_reduce_nowait() could not use tree reduction,
+        so use atomics instead:
+        each thread atomically writes to foo
+        each thread atomically writes to bar
+```
+
+The function can_block_reduce() is defined as follows:
+
+
+```
+int32_t can_block_reduce() {
+  int tid = GetThreadIdInTeam();
+  int nt = GetNumberOfOmpThreads(tid);
+  if (nt != blockDim.x)
+    return 0;
+  unsigned tnum = __ballot(1);
+  if (tnum != (~0x0)) {
+    return 0;
+  }
+  return 1;
+}
+```
+
+This function permits the use of the efficient block reduction algorithm
+using shuffles and shared memory (return 1) only if (a) all SIMT threads in
+a warp are active (i.e., number of threads in the parallel region is a
+multiple of 32) and (b) the number of threads in the parallel region
+(set by the num_threads clause) equals blockDim.x.
+
+If either of these preconditions is not true, each thread in the threadblock
+updates the global value using atomics.
+
+Atomics and compare-and-swap operations are expensive on many threaded
+architectures such as GPUs and we must avoid them completely.
+
+
+**Appendix: Implementation Details**
+
+
+```
+// Compiler generated function.
+reduceFn(ReduceData *a, ReduceData *b)
+  a->foo = a->foo + b->foo
+  a->bar = a->bar + b->bar
+
+// Compiler generated function.
+swapAndReduceFn(ReduceData *thread_private, int lane)
+  ReduceData *remote = new ReduceData()
+  remote->foo = shuffle_double(thread_private->foo, lane)
+  remote->bar = shuffle_double(thread_private->bar, lane)
+  reduceFn(thread_private, remote)
+
+// OMP runtime function.
+warpReduce_regular(ReduceData *thread_private, Fn *swapAndReduceFn):
+  offset = 16
+  while (offset > 0)
+    swapAndReduceFn(thread_private, offset)
+    offset /= 2
+
+// OMP runtime function.
+warpReduce_irregular():
+  ...
+
+// OMP runtime function.
+kmpc_reduce_warp(reduceData, swapAndReduceFn)
+  if all_lanes_active:
+    warpReduce_regular(reduceData, swapAndReduceFn)
+  else:
+    warpReduce_irregular(reduceData, swapAndReduceFn)
+  if in_simd_region:
+    // all done, reduce to global in simd lane 0
+    return 1
+  else if in_parallel_region:
+    // done reducing to one value per warp, now reduce across warps
+    return 3
+
+// OMP runtime function; one for each basic type.
+kmpc_reduce_block_double(double *a)
+  if lane == 0:
+    shared[wid] = *a
+  named_barrier(1, num_threads)
+  if wid == 0
+    block_reduce(shared)
+  if lane == 0
+    *a = shared[0]
+  named_barrier(1, num_threads)
+  if wid == 0 and lane == 0
+    return 1  // write back reduced result
+  else
+    return 0  // don't do anything
+
+```
+
+
+
+```
+// Compiler generated code.
+    1. Create private copies of variables: foo_p, bar_p
+    2. Each thread reduces the chunk of A and B assigned to it and writes
+       to foo_p and bar_p respectively.
+    3. ret = kmpc_reduce_warp(reduceData, swapAndReduceFn)
+    4. if ret == 1:
+        The master thread stores the reduced result in the globals.
+        foo += reduceData.foo; bar += reduceData.bar
+    5. else if ret == 3:
+        ret = block_reduce_double(reduceData.foo)
+        if ret == 1:
+          foo += reduceData.foo
+        ret = block_reduce_double(reduceData.bar)
+        if ret == 1:
+          bar += reduceData.bar
+```
+
+**Notes**
+
+    1. This scheme requires that the CUDA OMP runtime can call llvm generated
+       functions. This functionality now works.
+    2. If the user inlines the CUDA OMP runtime bitcode, all of the machinery
+       (including calls through function pointers) are optimized away.
+    3. If we are reducing multiple to multiple variables in a parallel region,
+       the reduce operations are all performed in warpReduce_[ir]regular(). This
+       results in more instructions in the loop and should result in fewer
+       stalls due to data dependencies.  Unfortunately we cannot do the same in
+       kmpc_reduce_block_double() without increasing shared memory usage.

libomptarget/deviceRTLs/nvptx/src/cancel.cu

@@ -0,0 +1,28 @@
+//===------ cancel.cu - NVPTX OpenMP cancel interface ------------ CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface to be used in the implementation of OpenMP cancel.
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+EXTERN int32_t __kmpc_cancellationpoint(kmp_Ident *loc, int32_t global_tid,
+                                        int32_t cancelVal) {
+  PRINT(LD_IO, "call kmpc_cancellationpoint(cancel val %d)\n", (int)cancelVal);
+  // disabled
+  return FALSE;
+}
+
+EXTERN int32_t __kmpc_cancel(kmp_Ident *loc, int32_t global_tid,
+                             int32_t cancelVal) {
+  PRINT(LD_IO, "call kmpc_cancel(cancel val %d)\n", (int)cancelVal);
+  // disabled
+  return FALSE;
+}

libomptarget/deviceRTLs/nvptx/src/critical.cu

@@ -0,0 +1,30 @@
+//===------ critical.cu - NVPTX OpenMP critical ------------------ CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of critical with KMPC interface
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdio.h>
+
+#include "omptarget-nvptx.h"
+
+EXTERN
+void __kmpc_critical(kmp_Ident *loc, int32_t global_tid,
+                     kmp_CriticalName *lck) {
+  PRINT0(LD_IO, "call to kmpc_critical()\n");
+  omp_set_lock((omp_lock_t *)lck);
+}
+
+EXTERN
+void __kmpc_end_critical(kmp_Ident *loc, int32_t global_tid,
+                         kmp_CriticalName *lck) {
+  PRINT0(LD_IO, "call to kmpc_end_critical()\n");
+  omp_unset_lock((omp_lock_t *)lck);
+}

libomptarget/deviceRTLs/nvptx/src/data_sharing.cu

@@ -0,0 +1,582 @@
+//===----- data_sharing.cu - NVPTX OpenMP debug utilities -------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of data sharing environments/
+//
+//===----------------------------------------------------------------------===//
+#include "omptarget-nvptx.h"
+#include <stdio.h>
+
+// Warp ID in the CUDA block
+INLINE static unsigned getWarpId() { return threadIdx.x / WARPSIZE; }
+// Lane ID in the CUDA warp.
+INLINE static unsigned getLaneId() { return threadIdx.x % WARPSIZE; }
+
+// Return true if this is the first active thread in the warp.
+INLINE static bool IsWarpMasterActiveThread() {
+  unsigned long long Mask = __ACTIVEMASK();
+  unsigned long long ShNum = WARPSIZE - (GetThreadIdInBlock() % WARPSIZE);
+  unsigned long long Sh = Mask << ShNum;
+  // Truncate Sh to the 32 lower bits
+  return (unsigned)Sh == 0;
+}
+// Return true if this is the master thread.
+INLINE static bool IsMasterThread(bool isSPMDExecutionMode) {
+  return !isSPMDExecutionMode && GetMasterThreadID() == GetThreadIdInBlock();
+}
+
+/// Return the provided size aligned to the size of a pointer.
+INLINE static size_t AlignVal(size_t Val) {
+  const size_t Align = (size_t)sizeof(void *);
+  if (Val & (Align - 1)) {
+    Val += Align;
+    Val &= ~(Align - 1);
+  }
+  return Val;
+}
+
+#define DSFLAG 0
+#define DSFLAG_INIT 0
+#define DSPRINT(_flag, _str, _args...)                                         \
+  {                                                                            \
+    if (_flag) {                                                               \
+      /*printf("(%d,%d) -> " _str, blockIdx.x, threadIdx.x, _args);*/          \
+    }                                                                          \
+  }
+#define DSPRINT0(_flag, _str)                                                  \
+  {                                                                            \
+    if (_flag) {                                                               \
+      /*printf("(%d,%d) -> " _str, blockIdx.x, threadIdx.x);*/                 \
+    }                                                                          \
+  }
+
+// Initialize the shared data structures. This is expected to be called for the
+// master thread and warp masters. \param RootS: A pointer to the root of the
+// data sharing stack. \param InitialDataSize: The initial size of the data in
+// the slot.
+EXTERN void
+__kmpc_initialize_data_sharing_environment(__kmpc_data_sharing_slot *rootS,
+                                           size_t InitialDataSize) {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
+  DSPRINT0(DSFLAG_INIT,
+           "Entering __kmpc_initialize_data_sharing_environment\n");
+
+  unsigned WID = getWarpId();
+  DSPRINT(DSFLAG_INIT, "Warp ID: %u\n", WID);
+
+  omptarget_nvptx_TeamDescr *teamDescr =
+      &omptarget_nvptx_threadPrivateContext->TeamContext();
+  __kmpc_data_sharing_slot *RootS =
+      teamDescr->RootS(WID, IsMasterThread(isSPMDMode()));
+
+  DataSharingState.SlotPtr[WID] = RootS;
+  DataSharingState.StackPtr[WID] = (void *)&RootS->Data[0];
+
+  // We don't need to initialize the frame and active threads.
+
+  DSPRINT(DSFLAG_INIT, "Initial data size: %08x \n", (unsigned)InitialDataSize);
+  DSPRINT(DSFLAG_INIT, "Root slot at: %016llx \n", (unsigned long long)RootS);
+  DSPRINT(DSFLAG_INIT, "Root slot data-end at: %016llx \n",
+          (unsigned long long)RootS->DataEnd);
+  DSPRINT(DSFLAG_INIT, "Root slot next at: %016llx \n",
+          (unsigned long long)RootS->Next);
+  DSPRINT(DSFLAG_INIT, "Shared slot ptr at: %016llx \n",
+          (unsigned long long)DataSharingState.SlotPtr[WID]);
+  DSPRINT(DSFLAG_INIT, "Shared stack ptr at: %016llx \n",
+          (unsigned long long)DataSharingState.StackPtr[WID]);
+
+  DSPRINT0(DSFLAG_INIT, "Exiting __kmpc_initialize_data_sharing_environment\n");
+}
+
+EXTERN void *__kmpc_data_sharing_environment_begin(
+    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
+    void **SavedSharedFrame, int32_t *SavedActiveThreads,
+    size_t SharingDataSize, size_t SharingDefaultDataSize,
+    int16_t IsOMPRuntimeInitialized) {
+
+  DSPRINT0(DSFLAG, "Entering __kmpc_data_sharing_environment_begin\n");
+
+  // If the runtime has been elided, used __shared__ memory for master-worker
+  // data sharing.
+  if (!IsOMPRuntimeInitialized)
+    return (void *)&DataSharingState;
+
+  DSPRINT(DSFLAG, "Data Size %016llx\n", (unsigned long long)SharingDataSize);
+  DSPRINT(DSFLAG, "Default Data Size %016llx\n",
+          (unsigned long long)SharingDefaultDataSize);
+
+  unsigned WID = getWarpId();
+  unsigned CurActiveThreads = __ACTIVEMASK();
+
+  __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
+  void *&StackP = DataSharingState.StackPtr[WID];
+  void * volatile &FrameP = DataSharingState.FramePtr[WID];
+  int32_t &ActiveT = DataSharingState.ActiveThreads[WID];
+
+  DSPRINT0(DSFLAG, "Save current slot/stack values.\n");
+  // Save the current values.
+  *SavedSharedSlot = SlotP;
+  *SavedSharedStack = StackP;
+  *SavedSharedFrame = FrameP;
+  *SavedActiveThreads = ActiveT;
+
+  DSPRINT(DSFLAG, "Warp ID: %u\n", WID);
+  DSPRINT(DSFLAG, "Saved slot ptr at: %016llx \n", (unsigned long long)SlotP);
+  DSPRINT(DSFLAG, "Saved stack ptr at: %016llx \n", (unsigned long long)StackP);
+  DSPRINT(DSFLAG, "Saved frame ptr at: %016llx \n", (long long)FrameP);
+  DSPRINT(DSFLAG, "Active threads: %08x \n", (unsigned)ActiveT);
+
+  // Only the warp active master needs to grow the stack.
+  if (IsWarpMasterActiveThread()) {
+    // Save the current active threads.
+    ActiveT = CurActiveThreads;
+
+    // Make sure we use aligned sizes to avoid rematerialization of data.
+    SharingDataSize = AlignVal(SharingDataSize);
+    // FIXME: The default data size can be assumed to be aligned?
+    SharingDefaultDataSize = AlignVal(SharingDefaultDataSize);
+
+    // Check if we have room for the data in the current slot.
+    const uintptr_t CurrentStartAddress = (uintptr_t)StackP;
+    const uintptr_t CurrentEndAddress = (uintptr_t)SlotP->DataEnd;
+    const uintptr_t RequiredEndAddress =
+        CurrentStartAddress + (uintptr_t)SharingDataSize;
+
+    DSPRINT(DSFLAG, "Data Size %016llx\n", (unsigned long long)SharingDataSize);
+    DSPRINT(DSFLAG, "Default Data Size %016llx\n",
+            (unsigned long long)SharingDefaultDataSize);
+    DSPRINT(DSFLAG, "Current Start Address %016llx\n",
+            (unsigned long long)CurrentStartAddress);
+    DSPRINT(DSFLAG, "Current End Address %016llx\n",
+            (unsigned long long)CurrentEndAddress);
+    DSPRINT(DSFLAG, "Required End Address %016llx\n",
+            (unsigned long long)RequiredEndAddress);
+    DSPRINT(DSFLAG, "Active Threads %08x\n", (unsigned)ActiveT);
+
+    // If we require a new slot, allocate it and initialize it (or attempt to
+    // reuse one). Also, set the shared stack and slot pointers to the new
+    // place. If we do not need to grow the stack, just adapt the stack and
+    // frame pointers.
+    if (CurrentEndAddress < RequiredEndAddress) {
+      size_t NewSize = (SharingDataSize > SharingDefaultDataSize)
+                           ? SharingDataSize
+                           : SharingDefaultDataSize;
+      __kmpc_data_sharing_slot *NewSlot = 0;
+
+      // Attempt to reuse an existing slot.
+      if (__kmpc_data_sharing_slot *ExistingSlot = SlotP->Next) {
+        uintptr_t ExistingSlotSize = (uintptr_t)ExistingSlot->DataEnd -
+                                     (uintptr_t)(&ExistingSlot->Data[0]);
+        if (ExistingSlotSize >= NewSize) {
+          DSPRINT(DSFLAG, "Reusing stack slot %016llx\n",
+                  (unsigned long long)ExistingSlot);
+          NewSlot = ExistingSlot;
+        } else {
+          DSPRINT(DSFLAG, "Cleaning up -failed reuse - %016llx\n",
+                  (unsigned long long)SlotP->Next);
+          free(ExistingSlot);
+        }
+      }
+
+      if (!NewSlot) {
+        NewSlot = (__kmpc_data_sharing_slot *)malloc(
+            sizeof(__kmpc_data_sharing_slot) + NewSize);
+        DSPRINT(DSFLAG, "New slot allocated %016llx (data size=%016llx)\n",
+                (unsigned long long)NewSlot, NewSize);
+      }
+
+      NewSlot->Next = 0;
+      NewSlot->DataEnd = &NewSlot->Data[NewSize];
+
+      SlotP->Next = NewSlot;
+      SlotP = NewSlot;
+      StackP = &NewSlot->Data[SharingDataSize];
+      FrameP = &NewSlot->Data[0];
+    } else {
+
+      // Clean up any old slot that we may still have. The slot producers, do
+      // not eliminate them because that may be used to return data.
+      if (SlotP->Next) {
+        DSPRINT(DSFLAG, "Cleaning up - old not required - %016llx\n",
+                (unsigned long long)SlotP->Next);
+        free(SlotP->Next);
+        SlotP->Next = 0;
+      }
+
+      FrameP = StackP;
+      StackP = (void *)RequiredEndAddress;
+    }
+  }
+
+  // FIXME: Need to see the impact of doing it here.
+  __threadfence_block();
+
+  DSPRINT0(DSFLAG, "Exiting __kmpc_data_sharing_environment_begin\n");
+
+  // All the threads in this warp get the frame they should work with.
+  return FrameP;
+}
+
+EXTERN void __kmpc_data_sharing_environment_end(
+    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
+    void **SavedSharedFrame, int32_t *SavedActiveThreads,
+    int32_t IsEntryPoint) {
+
+  DSPRINT0(DSFLAG, "Entering __kmpc_data_sharing_environment_end\n");
+
+  unsigned WID = getWarpId();
+
+  if (IsEntryPoint) {
+    if (IsWarpMasterActiveThread()) {
+      DSPRINT0(DSFLAG, "Doing clean up\n");
+
+      // The master thread cleans the saved slot, because this is an environment
+      // only for the master.
+      __kmpc_data_sharing_slot *S = IsMasterThread(isSPMDMode())
+                                        ? *SavedSharedSlot
+                                        : DataSharingState.SlotPtr[WID];
+
+      if (S->Next) {
+        free(S->Next);
+        S->Next = 0;
+      }
+    }
+
+    DSPRINT0(DSFLAG, "Exiting Exiting __kmpc_data_sharing_environment_end\n");
+    return;
+  }
+
+  int32_t CurActive = __ACTIVEMASK();
+
+  // Only the warp master can restore the stack and frame information, and only
+  // if there are no other threads left behind in this environment (i.e. the
+  // warp diverged and returns in different places). This only works if we
+  // assume that threads will converge right after the call site that started
+  // the environment.
+  if (IsWarpMasterActiveThread()) {
+    int32_t &ActiveT = DataSharingState.ActiveThreads[WID];
+
+    DSPRINT0(DSFLAG, "Before restoring the stack\n");
+    // Zero the bits in the mask. If it is still different from zero, then we
+    // have other threads that will return after the current ones.
+    ActiveT &= ~CurActive;
+
+    DSPRINT(DSFLAG, "Active threads: %08x; New mask: %08x\n",
+            (unsigned)CurActive, (unsigned)ActiveT);
+
+    if (!ActiveT) {
+      // No other active threads? Great, lets restore the stack.
+
+      __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
+      void *&StackP = DataSharingState.StackPtr[WID];
+      void * volatile &FrameP = DataSharingState.FramePtr[WID];
+
+      SlotP = *SavedSharedSlot;
+      StackP = *SavedSharedStack;
+      FrameP = *SavedSharedFrame;
+      ActiveT = *SavedActiveThreads;
+
+      DSPRINT(DSFLAG, "Restored slot ptr at: %016llx \n",
+              (unsigned long long)SlotP);
+      DSPRINT(DSFLAG, "Restored stack ptr at: %016llx \n",
+              (unsigned long long)StackP);
+      DSPRINT(DSFLAG, "Restored frame ptr at: %016llx \n",
+              (unsigned long long)FrameP);
+      DSPRINT(DSFLAG, "Active threads: %08x \n", (unsigned)ActiveT);
+    }
+  }
+
+  // FIXME: Need to see the impact of doing it here.
+  __threadfence_block();
+
+  DSPRINT0(DSFLAG, "Exiting __kmpc_data_sharing_environment_end\n");
+  return;
+}
+
+EXTERN void *
+__kmpc_get_data_sharing_environment_frame(int32_t SourceThreadID,
+                                          int16_t IsOMPRuntimeInitialized) {
+  DSPRINT0(DSFLAG, "Entering __kmpc_get_data_sharing_environment_frame\n");
+
+  // If the runtime has been elided, use __shared__ memory for master-worker
+  // data sharing.  We're reusing the statically allocated data structure
+  // that is used for standard data sharing.
+  if (!IsOMPRuntimeInitialized)
+    return (void *)&DataSharingState;
+
+  // Get the frame used by the requested thread.
+
+  unsigned SourceWID = SourceThreadID / WARPSIZE;
+
+  DSPRINT(DSFLAG, "Source  warp: %u\n", SourceWID);
+
+  void * volatile P = DataSharingState.FramePtr[SourceWID];
+  DSPRINT0(DSFLAG, "Exiting __kmpc_get_data_sharing_environment_frame\n");
+  return P;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Runtime functions for trunk data sharing scheme.
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE static void data_sharing_init_stack_common() {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
+  omptarget_nvptx_TeamDescr *teamDescr =
+      &omptarget_nvptx_threadPrivateContext->TeamContext();
+
+  for (int WID = 0; WID < WARPSIZE; WID++) {
+    __kmpc_data_sharing_slot *RootS = teamDescr->GetPreallocatedSlotAddr(WID);
+    DataSharingState.SlotPtr[WID] = RootS;
+    DataSharingState.StackPtr[WID] = (void *)&RootS->Data[0];
+  }
+}
+
+// Initialize data sharing data structure. This function needs to be called
+// once at the beginning of a data sharing context (coincides with the kernel
+// initialization). This function is called only by the MASTER thread of each
+// team in non-SPMD mode.
+EXTERN void __kmpc_data_sharing_init_stack() {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
+  // This function initializes the stack pointer with the pointer to the
+  // statically allocated shared memory slots. The size of a shared memory
+  // slot is pre-determined to be 256 bytes.
+  data_sharing_init_stack_common();
+  omptarget_nvptx_globalArgs.Init();
+}
+
+// Initialize data sharing data structure. This function needs to be called
+// once at the beginning of a data sharing context (coincides with the kernel
+// initialization). This function is called in SPMD mode only.
+EXTERN void __kmpc_data_sharing_init_stack_spmd() {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
+  // This function initializes the stack pointer with the pointer to the
+  // statically allocated shared memory slots. The size of a shared memory
+  // slot is pre-determined to be 256 bytes.
+  if (threadIdx.x == 0)
+    data_sharing_init_stack_common();
+
+  __threadfence_block();
+}
+
+INLINE static void* data_sharing_push_stack_common(size_t PushSize) {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
+
+  // Only warp active master threads manage the stack.
+  bool IsWarpMaster = (GetThreadIdInBlock() % WARPSIZE) == 0;
+
+  // Add worst-case padding to DataSize so that future stack allocations are
+  // correctly aligned.
+  const size_t Alignment = 8;
+  PushSize = (PushSize + (Alignment - 1)) / Alignment * Alignment;
+
+  // Frame pointer must be visible to all workers in the same warp.
+  const unsigned WID = getWarpId();
+  void *FrameP = 0;
+  int32_t CurActive = __ACTIVEMASK();
+
+  if (IsWarpMaster) {
+    // SlotP will point to either the shared memory slot or an existing
+    // global memory slot.
+    __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
+    void *&StackP = DataSharingState.StackPtr[WID];
+
+    // Check if we have room for the data in the current slot.
+    const uintptr_t StartAddress = (uintptr_t)StackP;
+    const uintptr_t EndAddress = (uintptr_t)SlotP->DataEnd;
+    const uintptr_t RequestedEndAddress = StartAddress + (uintptr_t)PushSize;
+
+    // If we requested more data than there is room for in the rest
+    // of the slot then we need to either re-use the next slot, if one exists,
+    // or create a new slot.
+    if (EndAddress < RequestedEndAddress) {
+      __kmpc_data_sharing_slot *NewSlot = 0;
+      size_t NewSize = PushSize;
+
+      // Allocate at least the default size for each type of slot.
+      // Master is a special case and even though there is only one thread,
+      // it can share more things with the workers. For uniformity, it uses
+      // the full size of a worker warp slot.
+      size_t DefaultSlotSize = DS_Worker_Warp_Slot_Size;
+      if (DefaultSlotSize > NewSize)
+        NewSize = DefaultSlotSize;
+      NewSlot = (__kmpc_data_sharing_slot *) SafeMalloc(
+          sizeof(__kmpc_data_sharing_slot) + NewSize,
+          "Global memory slot allocation.");
+
+      NewSlot->Next = 0;
+      NewSlot->Prev = SlotP;
+      NewSlot->PrevSlotStackPtr = StackP;
+      NewSlot->DataEnd = &NewSlot->Data[0] + NewSize;
+
+      // Make previous slot point to the newly allocated slot.
+      SlotP->Next = NewSlot;
+      // The current slot becomes the new slot.
+      SlotP = NewSlot;
+      // The stack pointer always points to the next free stack frame.
+      StackP = &NewSlot->Data[0] + PushSize;
+      // The frame pointer always points to the beginning of the frame.
+      FrameP = DataSharingState.FramePtr[WID] = &NewSlot->Data[0];
+    } else {
+      // Add the data chunk to the current slot. The frame pointer is set to
+      // point to the start of the new frame held in StackP.
+      FrameP = DataSharingState.FramePtr[WID] = StackP;
+      // Reset stack pointer to the requested address.
+      StackP = (void *)RequestedEndAddress;
+    }
+  }
+  // Get address from lane 0.
+  ((int *)&FrameP)[0] = __SHFL_SYNC(CurActive, ((int *)&FrameP)[0], 0);
+  if (sizeof(FrameP) == 8)
+    ((int *)&FrameP)[1] = __SHFL_SYNC(CurActive, ((int *)&FrameP)[1], 0);
+
+  return FrameP;
+}
+
+EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t DataSize,
+                                                      int16_t UseSharedMemory) {
+  return data_sharing_push_stack_common(DataSize);
+}
+
+// Called at the time of the kernel initialization. This is used to initilize
+// the list of references to shared variables and to pre-allocate global storage
+// for holding the globalized variables.
+//
+// By default the globalized variables are stored in global memory. If the
+// UseSharedMemory is set to true, the runtime will attempt to use shared memory
+// as long as the size requested fits the pre-allocated size.
+EXTERN void *__kmpc_data_sharing_push_stack(size_t DataSize,
+                                            int16_t UseSharedMemory) {
+  // Compute the total memory footprint of the requested data.
+  // The master thread requires a stack only for itself. A worker
+  // thread (which at this point is a warp master) will require
+  // space for the variables of each thread in the warp,
+  // i.e. one DataSize chunk per warp lane.
+  // TODO: change WARPSIZE to the number of active threads in the warp.
+  size_t PushSize = (isRuntimeUninitialized() || IsMasterThread(isSPMDMode()))
+                        ? DataSize
+                        : WARPSIZE * DataSize;
+
+  // Compute the start address of the frame of each thread in the warp.
+  uintptr_t FrameStartAddress =
+      (uintptr_t) data_sharing_push_stack_common(PushSize);
+  FrameStartAddress += (uintptr_t) (getLaneId() * DataSize);
+  return (void *)FrameStartAddress;
+}
+
+// Pop the stack and free any memory which can be reclaimed.
+//
+// When the pop operation removes the last global memory slot,
+// reclaim all outstanding global memory slots since it is
+// likely we have reached the end of the kernel.
+EXTERN void __kmpc_data_sharing_pop_stack(void *FrameStart) {
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
+
+  __threadfence_block();
+
+  if (GetThreadIdInBlock() % WARPSIZE == 0) {
+    unsigned WID = getWarpId();
+
+    // Current slot
+    __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
+
+    // Pointer to next available stack.
+    void *&StackP = DataSharingState.StackPtr[WID];
+
+    // Pop the frame.
+    StackP = FrameStart;
+
+    // If the current slot is empty, we need to free the slot after the
+    // pop.
+    bool SlotEmpty = (StackP == &SlotP->Data[0]);
+
+    if (SlotEmpty && SlotP->Prev) {
+      // Before removing the slot we need to reset StackP.
+      StackP = SlotP->PrevSlotStackPtr;
+
+      // Remove the slot.
+      SlotP = SlotP->Prev;
+      SafeFree(SlotP->Next, "Free slot.");
+      SlotP->Next = 0;
+    }
+  }
+}
+
+// Begin a data sharing context. Maintain a list of references to shared
+// variables. This list of references to shared variables will be passed
+// to one or more threads.
+// In L0 data sharing this is called by master thread.
+// In L1 data sharing this is called by active warp master thread.
+EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs) {
+  omptarget_nvptx_globalArgs.EnsureSize(nArgs);
+  *GlobalArgs = omptarget_nvptx_globalArgs.GetArgs();
+}
+
+// End a data sharing context. There is no need to have a list of refs
+// to shared variables because the context in which those variables were
+// shared has now ended. This should clean-up the list of references only
+// without affecting the actual global storage of the variables.
+// In L0 data sharing this is called by master thread.
+// In L1 data sharing this is called by active warp master thread.
+EXTERN void __kmpc_end_sharing_variables() {
+  omptarget_nvptx_globalArgs.DeInit();
+}
+
+// This function will return a list of references to global variables. This
+// is how the workers will get a reference to the globalized variable. The
+// members of this list will be passed to the outlined parallel function
+// preserving the order.
+// Called by all workers.
+EXTERN void __kmpc_get_shared_variables(void ***GlobalArgs) {
+  *GlobalArgs = omptarget_nvptx_globalArgs.GetArgs();
+}
+
+// This function is used to init static memory manager. This manager is used to
+// manage statically allocated global memory. This memory is allocated by the
+// compiler and used to correctly implement globalization of the variables in
+// target, teams and distribute regions.
+EXTERN void __kmpc_get_team_static_memory(int16_t isSPMDExecutionMode,
+                                          const void *buf, size_t size,
+                                          int16_t is_shared,
+                                          const void **frame) {
+  if (is_shared) {
+    *frame = buf;
+    return;
+  }
+  if (isSPMDExecutionMode) {
+    if (GetThreadIdInBlock() == 0) {
+      *frame = omptarget_nvptx_simpleMemoryManager.Acquire(buf, size);
+    }
+    // FIXME: use __syncthreads instead when the function copy is fixed in LLVM.
+    __SYNCTHREADS();
+    return;
+  }
+  ASSERT0(LT_FUSSY, GetThreadIdInBlock() == GetMasterThreadID(),
+          "Must be called only in the target master thread.");
+  *frame = omptarget_nvptx_simpleMemoryManager.Acquire(buf, size);
+  __threadfence();
+}
+
+EXTERN void __kmpc_restore_team_static_memory(int16_t isSPMDExecutionMode,
+                                              int16_t is_shared) {
+  if (is_shared)
+    return;
+  if (isSPMDExecutionMode) {
+    // FIXME: use __syncthreads instead when the function copy is fixed in LLVM.
+    __SYNCTHREADS();
+    if (GetThreadIdInBlock() == 0) {
+      omptarget_nvptx_simpleMemoryManager.Release();
+    }
+    return;
+  }
+  __threadfence();
+  ASSERT0(LT_FUSSY, GetThreadIdInBlock() == GetMasterThreadID(),
+          "Must be called only in the target master thread.");
+  omptarget_nvptx_simpleMemoryManager.Release();
+}
+

libomptarget/deviceRTLs/nvptx/src/debug.h

@@ -0,0 +1,289 @@
+//===------------- debug.h - NVPTX OpenMP debug macros ----------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains debug macros to be used in the application.
+//
+//   Usage guide
+//
+//   PRINT0(flag, str)        : if debug flag is on, print (no arguments)
+//   PRINT(flag, str, args)   : if debug flag is on, print (arguments)
+//   DON(flag)                : return true if debug flag is on
+//
+//   ASSERT(flag, cond, str, args): if test flag is on, test the condition
+//                                  if the condition is false, print str+args
+//          and assert.
+//          CAUTION: cond may be evaluate twice
+//   AON(flag)                     : return true if test flag is on
+//
+//   WARNING(flag, str, args)      : if warning flag is on, print the warning
+//   WON(flag)                     : return true if warning flag is on
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _OMPTARGET_NVPTX_DEBUG_H_
+#define _OMPTARGET_NVPTX_DEBUG_H_
+
+////////////////////////////////////////////////////////////////////////////////
+// set desired level of debugging
+////////////////////////////////////////////////////////////////////////////////
+
+#define LD_SET_NONE 0ULL /* none */
+#define LD_SET_ALL -1ULL /* all */
+
+// pos 1
+#define LD_SET_LOOP 0x1ULL  /* basic loop */
+#define LD_SET_LOOPD 0x2ULL /* basic loop */
+#define LD_SET_PAR 0x4ULL   /* basic parallel */
+#define LD_SET_PARD 0x8ULL  /* basic parallel */
+
+// pos 2
+#define LD_SET_SYNC 0x10ULL  /* sync info */
+#define LD_SET_SYNCD 0x20ULL /* sync info */
+#define LD_SET_WAIT 0x40ULL  /* state when waiting */
+#define LD_SET_TASK 0x80ULL  /* print task info (high level) */
+
+// pos 3
+#define LD_SET_IO 0x100ULL     /* big region io (excl atomic) */
+#define LD_SET_IOD 0x200ULL    /* big region io (excl atomic) */
+#define LD_SET_ENV 0x400ULL    /* env info */
+#define LD_SET_CANCEL 0x800ULL /* print cancel info */
+
+// pos 4
+#define LD_SET_MEM 0x1000ULL /* malloc / free */
+
+////////////////////////////////////////////////////////////////////////////////
+// set the desired flags to print selected output.
+
+// these are some examples of possible definitions that can be used for
+// debugging.
+//#define OMPTARGET_NVPTX_DEBUG (LD_SET_ALL)
+//#define OMPTARGET_NVPTX_DEBUG (LD_SET_LOOP) // limit to loop printfs to save
+// on cuda buffer
+//#define OMPTARGET_NVPTX_DEBUG (LD_SET_IO)
+//#define OMPTARGET_NVPTX_DEBUG (LD_SET_IO | LD_SET_ENV)
+//#define OMPTARGET_NVPTX_DEBUG (LD_SET_PAR)
+
+#ifndef OMPTARGET_NVPTX_DEBUG
+#define OMPTARGET_NVPTX_DEBUG LD_SET_NONE
+#elif OMPTARGET_NVPTX_DEBUG
+#warning debug is used, not good for measurements
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// set desired level of asserts
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// available flags
+
+#define LT_SET_NONE 0x0 /* unsafe */
+#define LT_SET_SAFETY                                                          \
+  0x1 /* check malloc type of stuff, input at creation, cheap */
+#define LT_SET_INPUT 0x2 /* check also all runtime inputs */
+#define LT_SET_FUSSY 0x4 /* fussy checks, expensive */
+
+////////////////////////////////////////////////////////////////////////////////
+// set the desired flags
+
+#ifndef OMPTARGET_NVPTX_TEST
+#if OMPTARGET_NVPTX_DEBUG
+#define OMPTARGET_NVPTX_TEST (LT_SET_FUSSY)
+#else
+#define OMPTARGET_NVPTX_TEST (LT_SET_SAFETY)
+#endif
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// set desired level of warnings
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// available flags
+
+#define LW_SET_ALL -1
+#define LW_SET_NONE 0x0
+#define LW_SET_ENV 0x1
+#define LW_SET_INPUT 0x2
+#define LW_SET_FUSSY 0x4
+
+////////////////////////////////////////////////////////////////////////////////
+// set the desired flags
+
+#if OMPTARGET_NVPTX_DEBUG
+#define OMPTARGET_NVPTX_WARNING (LW_SET_NONE)
+#else
+#define OMPTARGET_NVPTX_WARNING (LW_SET_FUSSY)
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// implemtation for debug
+////////////////////////////////////////////////////////////////////////////////
+
+#if OMPTARGET_NVPTX_DEBUG || OMPTARGET_NVPTX_TEST || OMPTARGET_NVPTX_WARNING
+#include <stdio.h>
+#include "option.h"
+
+template <typename... Arguments>
+NOINLINE static void log(const char *fmt, Arguments... parameters) {
+  printf(fmt, (int)blockIdx.x, (int)threadIdx.x, (int)(threadIdx.x / WARPSIZE),
+         (int)(threadIdx.x & 0x1F), parameters...);
+}
+
+#endif
+#if OMPTARGET_NVPTX_TEST
+#include <assert.h>
+
+template <typename... Arguments>
+NOINLINE static void check(bool cond, const char *fmt,
+                           Arguments... parameters) {
+  if (!cond)
+    printf(fmt, (int)blockIdx.x, (int)threadIdx.x,
+           (int)(threadIdx.x / WARPSIZE), (int)(threadIdx.x & 0x1F),
+           parameters...);
+  assert(cond);
+}
+
+NOINLINE static void check(bool cond) { assert(cond); }
+#endif
+
+// set flags that are tested (inclusion properties)
+
+#define LD_ALL (LD_SET_ALL)
+
+#define LD_LOOP (LD_SET_LOOP | LD_SET_LOOPD)
+#define LD_LOOPD (LD_SET_LOOPD)
+#define LD_PAR (LD_SET_PAR | LD_SET_PARD)
+#define LD_PARD (LD_SET_PARD)
+
+// pos 2
+#define LD_SYNC (LD_SET_SYNC | LD_SET_SYNCD)
+#define LD_SYNCD (LD_SET_SYNCD)
+#define LD_WAIT (LD_SET_WAIT)
+#define LD_TASK (LD_SET_TASK)
+
+// pos 3
+#define LD_IO (LD_SET_IO | LD_SET_IOD)
+#define LD_IOD (LD_SET_IOD)
+#define LD_ENV (LD_SET_ENV)
+#define LD_CANCEL (LD_SET_CANCEL)
+
+// pos 3
+#define LD_MEM (LD_SET_MEM)
+
+// implement
+#if OMPTARGET_NVPTX_DEBUG
+
+#define DON(_flag) ((unsigned)(OMPTARGET_NVPTX_DEBUG) & (_flag))
+
+#define PRINT0(_flag, _str)                                                    \
+  {                                                                            \
+    if (omptarget_device_environment.debug_level && DON(_flag)) {              \
+      log("<b %2d, t %4d, w %2d, l %2d>: " _str);                              \
+    }                                                                          \
+  }
+
+#define PRINT(_flag, _str, _args...)                                           \
+  {                                                                            \
+    if (omptarget_device_environment.debug_level && DON(_flag)) {              \
+      log("<b %2d, t %4d, w %2d, l %2d>: " _str, _args);                       \
+    }                                                                          \
+  }
+#else
+
+#define DON(_flag) (FALSE)
+#define PRINT0(flag, str)
+#define PRINT(flag, str, _args...)
+
+#endif
+
+// for printing without worring about precision, pointers...
+#define P64(_x) ((unsigned long long)(_x))
+
+////////////////////////////////////////////////////////////////////////////////
+// early defs for test
+////////////////////////////////////////////////////////////////////////////////
+
+#define LT_SAFETY (LT_SET_SAFETY | LT_SET_INPUT | LT_SET_FUSSY)
+#define LT_INPUT (LT_SET_INPUT | LT_SET_FUSSY)
+#define LT_FUSSY (LT_SET_FUSSY)
+
+#if OMPTARGET_NVPTX_TEST == LT_SET_SAFETY
+
+#define TON(_flag) ((OMPTARGET_NVPTX_TEST) & (_flag))
+#define ASSERT0(_flag, _cond, _str)                                            \
+  {                                                                            \
+    if (TON(_flag)) {                                                          \
+      check(_cond);                                                            \
+    }                                                                          \
+  }
+#define ASSERT(_flag, _cond, _str, _args...)                                   \
+  {                                                                            \
+    if (TON(_flag)) {                                                          \
+      check(_cond);                                                            \
+    }                                                                          \
+  }
+
+#elif OMPTARGET_NVPTX_TEST >= LT_SET_INPUT
+
+#define TON(_flag) ((OMPTARGET_NVPTX_TEST) & (_flag))
+#define ASSERT0(_flag, _cond, _str)                                            \
+  {                                                                            \
+    if (TON(_flag)) {                                                          \
+      check((_cond), "<b %3d, t %4d, w %2d, l %2d> ASSERT: " _str "\n");       \
+    }                                                                          \
+  }
+#define ASSERT(_flag, _cond, _str, _args...)                                   \
+  {                                                                            \
+    if (TON(_flag)) {                                                          \
+      check((_cond), "<b %3d, t %4d, w %2d, l %d2> ASSERT: " _str "\n",        \
+            _args);                                                            \
+    }                                                                          \
+  }
+
+#else
+
+#define TON(_flag) (FALSE)
+#define ASSERT0(_flag, _cond, _str)
+#define ASSERT(_flag, _cond, _str, _args...)
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// early defs for warning
+
+#define LW_ALL (LW_SET_ALL)
+#define LW_ENV (LW_SET_FUSSY | LW_SET_INPUT | LW_SET_ENV)
+#define LW_INPUT (LW_SET_FUSSY | LW_SET_INPUT)
+#define LW_FUSSY (LW_SET_FUSSY)
+
+#if OMPTARGET_NVPTX_WARNING
+
+#define WON(_flag) ((OMPTARGET_NVPTX_WARNING) & (_flag))
+#define WARNING0(_flag, _str)                                                  \
+  {                                                                            \
+    if (WON(_flag)) {                                                          \
+      log("<b %2d, t %4d, w %2d, l %2d> WARNING: " _str);                      \
+    }                                                                          \
+  }
+#define WARNING(_flag, _str, _args...)                                         \
+  {                                                                            \
+    if (WON(_flag)) {                                                          \
+      log("<b %2d, t %4d, w %2d, l %2d> WARNING: " _str, _args);               \
+    }                                                                          \
+  }
+
+#else
+
+#define WON(_flag) (FALSE)
+#define WARNING0(_flag, _str)
+#define WARNING(_flag, _str, _args...)
+
+#endif
+
+#endif

libomptarget/deviceRTLs/nvptx/src/interface.h

@@ -0,0 +1,564 @@
+//===------- interface.h - NVPTX OpenMP interface definitions ---- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains debug macros to be used in the application.
+//
+//  This file contains all the definitions that are relevant to
+//  the interface. The first section contains the interface as
+//  declared by OpenMP.  A second section includes library private calls
+//  (mostly debug, temporary?) The third section includes the compiler
+//  specific interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _INTERFACES_H_
+#define _INTERFACES_H_
+
+#include "option.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// OpenMP interface
+////////////////////////////////////////////////////////////////////////////////
+
+typedef uint32_t omp_lock_t;      /* arbitrary type of the right length */
+typedef uint64_t omp_nest_lock_t; /* arbitrary type of the right length */
+
+typedef enum omp_sched_t {
+  omp_sched_static = 1,  /* chunkSize >0 */
+  omp_sched_dynamic = 2, /* chunkSize >0 */
+  omp_sched_guided = 3,  /* chunkSize >0 */
+  omp_sched_auto = 4,    /* no chunkSize */
+} omp_sched_t;
+
+typedef enum omp_proc_bind_t {
+  omp_proc_bind_false = 0,
+  omp_proc_bind_true = 1,
+  omp_proc_bind_master = 2,
+  omp_proc_bind_close = 3,
+  omp_proc_bind_spread = 4
+} omp_proc_bind_t;
+
+EXTERN double omp_get_wtick(void);
+EXTERN double omp_get_wtime(void);
+
+EXTERN void omp_set_num_threads(int num);
+EXTERN int omp_get_num_threads(void);
+EXTERN int omp_get_max_threads(void);
+EXTERN int omp_get_thread_limit(void);
+EXTERN int omp_get_thread_num(void);
+EXTERN int omp_get_num_procs(void);
+EXTERN int omp_in_parallel(void);
+EXTERN int omp_in_final(void);
+EXTERN void omp_set_dynamic(int flag);
+EXTERN int omp_get_dynamic(void);
+EXTERN void omp_set_nested(int flag);
+EXTERN int omp_get_nested(void);
+EXTERN void omp_set_max_active_levels(int level);
+EXTERN int omp_get_max_active_levels(void);
+EXTERN int omp_get_level(void);
+EXTERN int omp_get_active_level(void);
+EXTERN int omp_get_ancestor_thread_num(int level);
+EXTERN int omp_get_team_size(int level);
+
+EXTERN void omp_init_lock(omp_lock_t *lock);
+EXTERN void omp_init_nest_lock(omp_nest_lock_t *lock);
+EXTERN void omp_destroy_lock(omp_lock_t *lock);
+EXTERN void omp_destroy_nest_lock(omp_nest_lock_t *lock);
+EXTERN void omp_set_lock(omp_lock_t *lock);
+EXTERN void omp_set_nest_lock(omp_nest_lock_t *lock);
+EXTERN void omp_unset_lock(omp_lock_t *lock);
+EXTERN void omp_unset_nest_lock(omp_nest_lock_t *lock);
+EXTERN int omp_test_lock(omp_lock_t *lock);
+EXTERN int omp_test_nest_lock(omp_nest_lock_t *lock);
+
+EXTERN void omp_get_schedule(omp_sched_t *kind, int *modifier);
+EXTERN void omp_set_schedule(omp_sched_t kind, int modifier);
+EXTERN omp_proc_bind_t omp_get_proc_bind(void);
+EXTERN int omp_get_cancellation(void);
+EXTERN void omp_set_default_device(int deviceId);
+EXTERN int omp_get_default_device(void);
+EXTERN int omp_get_num_devices(void);
+EXTERN int omp_get_num_teams(void);
+EXTERN int omp_get_team_num(void);
+EXTERN int omp_is_initial_device(void);
+EXTERN int omp_get_initial_device(void);
+EXTERN int omp_get_max_task_priority(void);
+
+////////////////////////////////////////////////////////////////////////////////
+// file below is swiped from kmpc host interface
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// kmp specifc types
+////////////////////////////////////////////////////////////////////////////////
+
+typedef enum kmp_sched_t {
+  kmp_sched_static_chunk = 33,
+  kmp_sched_static_nochunk = 34,
+  kmp_sched_dynamic = 35,
+  kmp_sched_guided = 36,
+  kmp_sched_runtime = 37,
+  kmp_sched_auto = 38,
+
+  kmp_sched_static_balanced_chunk = 45,
+
+  kmp_sched_static_ordered = 65,
+  kmp_sched_static_nochunk_ordered = 66,
+  kmp_sched_dynamic_ordered = 67,
+  kmp_sched_guided_ordered = 68,
+  kmp_sched_runtime_ordered = 69,
+  kmp_sched_auto_ordered = 70,
+
+  kmp_sched_distr_static_chunk = 91,
+  kmp_sched_distr_static_nochunk = 92,
+  kmp_sched_distr_static_chunk_sched_static_chunkone = 93,
+
+  kmp_sched_default = kmp_sched_static_nochunk,
+  kmp_sched_unordered_first = kmp_sched_static_chunk,
+  kmp_sched_unordered_last = kmp_sched_auto,
+  kmp_sched_ordered_first = kmp_sched_static_ordered,
+  kmp_sched_ordered_last = kmp_sched_auto_ordered,
+  kmp_sched_distribute_first = kmp_sched_distr_static_chunk,
+  kmp_sched_distribute_last =
+      kmp_sched_distr_static_chunk_sched_static_chunkone,
+
+  /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
+   * Since we need to distinguish the three possible cases (no modifier,
+   * monotonic modifier, nonmonotonic modifier), we need separate bits for
+   * each modifier. The absence of monotonic does not imply nonmonotonic,
+   * especially since 4.5 says that the behaviour of the "no modifier" case
+   * is implementation defined in 4.5, but will become "nonmonotonic" in 5.0.
+   *
+   * Since we're passing a full 32 bit value, we can use a couple of high
+   * bits for these flags; out of paranoia we avoid the sign bit.
+   *
+   * These modifiers can be or-ed into non-static schedules by the compiler
+   * to pass the additional information. They will be stripped early in the
+   * processing in __kmp_dispatch_init when setting up schedules, so
+   * most of the code won't ever see schedules with these bits set.
+   */
+  kmp_sched_modifier_monotonic = (1 << 29),
+  /**< Set if the monotonic schedule modifier was present */
+  kmp_sched_modifier_nonmonotonic = (1 << 30),
+/**< Set if the nonmonotonic schedule modifier was present */
+
+#define SCHEDULE_WITHOUT_MODIFIERS(s)                                          \
+  (enum kmp_sched_t)(                                                          \
+      (s) & ~(kmp_sched_modifier_nonmonotonic | kmp_sched_modifier_monotonic))
+#define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sched_modifier_monotonic) != 0)
+#define SCHEDULE_HAS_NONMONOTONIC(s)                                           \
+  (((s)&kmp_sched_modifier_nonmonotonic) != 0)
+#define SCHEDULE_HAS_NO_MODIFIERS(s)                                           \
+  (((s) & (kmp_sched_modifier_nonmonotonic | kmp_sched_modifier_monotonic)) == \
+   0)
+
+} kmp_sched_t;
+
+/*!
+ * Enum for accesseing the reserved_2 field of the ident_t struct below.
+ */
+enum {
+  /*! Bit set to 1 when in SPMD mode. */
+  KMP_IDENT_SPMD_MODE = 0x01,
+  /*! Bit set to 1 when a simplified runtime is used. */
+  KMP_IDENT_SIMPLE_RT_MODE = 0x02,
+};
+
+/*!
+ * The ident structure that describes a source location.
+ * The struct is identical to the one in the kmp.h file.
+ * We maintain the same data structure for compatibility.
+ */
+typedef int kmp_int32;
+typedef struct ident {
+  kmp_int32 reserved_1; /**<  might be used in Fortran; see above  */
+  kmp_int32 flags; /**<  also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC
+                      identifies this union member  */
+  kmp_int32 reserved_2; /**<  not really used in Fortran any more; see above */
+  kmp_int32 reserved_3; /**<  source[4] in Fortran, do not use for C++  */
+  char const *psource; /**<  String describing the source location.
+                       The string is composed of semi-colon separated fields
+                       which describe the source file, the function and a pair
+                       of line numbers that delimit the construct. */
+} ident_t;
+
+// parallel defs
+typedef ident_t kmp_Ident;
+typedef void (*kmp_ParFctPtr)(int32_t *global_tid, int32_t *bound_tid, ...);
+typedef void (*kmp_ReductFctPtr)(void *lhsData, void *rhsData);
+typedef void (*kmp_InterWarpCopyFctPtr)(void *src, int32_t warp_num);
+typedef void (*kmp_ShuffleReductFctPtr)(void *rhsData, int16_t lane_id,
+                                        int16_t lane_offset,
+                                        int16_t shortCircuit);
+typedef void (*kmp_CopyToScratchpadFctPtr)(void *reduceData, void *scratchpad,
+                                           int32_t index, int32_t width);
+typedef void (*kmp_LoadReduceFctPtr)(void *reduceData, void *scratchpad,
+                                     int32_t index, int32_t width,
+                                     int32_t reduce);
+
+// task defs
+typedef struct kmp_TaskDescr kmp_TaskDescr;
+typedef int32_t (*kmp_TaskFctPtr)(int32_t global_tid, kmp_TaskDescr *taskDescr);
+typedef struct kmp_TaskDescr {
+  void *sharedPointerTable;   // ptr to a table of shared var ptrs
+  kmp_TaskFctPtr sub;         // task subroutine
+  int32_t partId;             // unused
+  kmp_TaskFctPtr destructors; // destructor of c++ first private
+} kmp_TaskDescr;
+// task dep defs
+#define KMP_TASKDEP_IN 0x1u
+#define KMP_TASKDEP_OUT 0x2u
+typedef struct kmp_TaskDep_Public {
+  void *addr;
+  size_t len;
+  uint8_t flags; // bit 0: in, bit 1: out
+} kmp_TaskDep_Public;
+
+// flags that interpret the interface part of tasking flags
+#define KMP_TASK_IS_TIED 0x1
+#define KMP_TASK_FINAL 0x2
+#define KMP_TASK_MERGED_IF0 0x4 /* unused */
+#define KMP_TASK_DESTRUCTOR_THUNK 0x8
+
+// flags for task setup return
+#define KMP_CURRENT_TASK_NOT_SUSPENDED 0
+#define KMP_CURRENT_TASK_SUSPENDED 1
+
+// sync defs
+typedef int32_t kmp_CriticalName[8];
+
+////////////////////////////////////////////////////////////////////////////////
+// flags for kstate (all bits initially off)
+////////////////////////////////////////////////////////////////////////////////
+
+// first 2 bits used by kmp_Reduction (defined in kmp_reduction.cpp)
+#define KMP_REDUCTION_MASK 0x3
+#define KMP_SKIP_NEXT_CALL 0x4
+#define KMP_SKIP_NEXT_CANCEL_BARRIER 0x8
+
+////////////////////////////////////////////////////////////////////////////////
+// data
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// external interface
+////////////////////////////////////////////////////////////////////////////////
+
+// query
+EXTERN int32_t __kmpc_global_num_threads(kmp_Ident *loc); // missing
+EXTERN int32_t __kmpc_bound_thread_num(kmp_Ident *loc);   // missing
+EXTERN int32_t __kmpc_bound_num_threads(kmp_Ident *loc);  // missing
+EXTERN int32_t __kmpc_in_parallel(kmp_Ident *loc);        // missing
+
+// parallel
+EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc);
+EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t global_tid,
+                                    int32_t num_threads);
+// simd
+EXTERN void __kmpc_push_simd_limit(kmp_Ident *loc, int32_t global_tid,
+                                   int32_t simd_limit);
+// aee ... not supported
+// EXTERN void __kmpc_fork_call(kmp_Ident *loc, int32_t argc, kmp_ParFctPtr
+// microtask, ...);
+EXTERN void __kmpc_serialized_parallel(kmp_Ident *loc, uint32_t global_tid);
+EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc,
+                                           uint32_t global_tid);
+EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid);
+
+// proc bind
+EXTERN void __kmpc_push_proc_bind(kmp_Ident *loc, uint32_t global_tid,
+                                  int proc_bind);
+EXTERN int omp_get_num_places(void);
+EXTERN int omp_get_place_num_procs(int place_num);
+EXTERN void omp_get_place_proc_ids(int place_num, int *ids);
+EXTERN int omp_get_place_num(void);
+EXTERN int omp_get_partition_num_places(void);
+EXTERN void omp_get_partition_place_nums(int *place_nums);
+
+// for static (no chunk or chunk)
+EXTERN void __kmpc_for_static_init_4(kmp_Ident *loc, int32_t global_tid,
+                                     int32_t sched, int32_t *plastiter,
+                                     int32_t *plower, int32_t *pupper,
+                                     int32_t *pstride, int32_t incr,
+                                     int32_t chunk);
+EXTERN void __kmpc_for_static_init_4u(kmp_Ident *loc, int32_t global_tid,
+                                      int32_t sched, int32_t *plastiter,
+                                      uint32_t *plower, uint32_t *pupper,
+                                      int32_t *pstride, int32_t incr,
+                                      int32_t chunk);
+EXTERN void __kmpc_for_static_init_8(kmp_Ident *loc, int32_t global_tid,
+                                     int32_t sched, int32_t *plastiter,
+                                     int64_t *plower, int64_t *pupper,
+                                     int64_t *pstride, int64_t incr,
+                                     int64_t chunk);
+EXTERN void __kmpc_for_static_init_8u(kmp_Ident *loc, int32_t global_tid,
+                                      int32_t sched, int32_t *plastiter1,
+                                      uint64_t *plower, uint64_t *pupper,
+                                      int64_t *pstride, int64_t incr,
+                                      int64_t chunk);
+EXTERN
+void __kmpc_for_static_init_4_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                          int32_t sched, int32_t *plastiter,
+                                          int32_t *plower, int32_t *pupper,
+                                          int32_t *pstride, int32_t incr,
+                                          int32_t chunk);
+EXTERN
+void __kmpc_for_static_init_4u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                           int32_t sched, int32_t *plastiter,
+                                           uint32_t *plower, uint32_t *pupper,
+                                           int32_t *pstride, int32_t incr,
+                                           int32_t chunk);
+EXTERN
+void __kmpc_for_static_init_8_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                          int32_t sched, int32_t *plastiter,
+                                          int64_t *plower, int64_t *pupper,
+                                          int64_t *pstride, int64_t incr,
+                                          int64_t chunk);
+EXTERN
+void __kmpc_for_static_init_8u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                           int32_t sched, int32_t *plastiter1,
+                                           uint64_t *plower, uint64_t *pupper,
+                                           int64_t *pstride, int64_t incr,
+                                           int64_t chunk);
+EXTERN
+void __kmpc_for_static_init_4_simple_generic(kmp_Ident *loc,
+                                             int32_t global_tid, int32_t sched,
+                                             int32_t *plastiter,
+                                             int32_t *plower, int32_t *pupper,
+                                             int32_t *pstride, int32_t incr,
+                                             int32_t chunk);
+EXTERN
+void __kmpc_for_static_init_4u_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t sched, int32_t *plastiter,
+    uint32_t *plower, uint32_t *pupper, int32_t *pstride, int32_t incr,
+    int32_t chunk);
+EXTERN
+void __kmpc_for_static_init_8_simple_generic(kmp_Ident *loc,
+                                             int32_t global_tid, int32_t sched,
+                                             int32_t *plastiter,
+                                             int64_t *plower, int64_t *pupper,
+                                             int64_t *pstride, int64_t incr,
+                                             int64_t chunk);
+EXTERN
+void __kmpc_for_static_init_8u_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t sched, int32_t *plastiter1,
+    uint64_t *plower, uint64_t *pupper, int64_t *pstride, int64_t incr,
+    int64_t chunk);
+
+EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid);
+
+// for dynamic
+EXTERN void __kmpc_dispatch_init_4(kmp_Ident *loc, int32_t global_tid,
+                                   int32_t sched, int32_t lower, int32_t upper,
+                                   int32_t incr, int32_t chunk);
+EXTERN void __kmpc_dispatch_init_4u(kmp_Ident *loc, int32_t global_tid,
+                                    int32_t sched, uint32_t lower,
+                                    uint32_t upper, int32_t incr,
+                                    int32_t chunk);
+EXTERN void __kmpc_dispatch_init_8(kmp_Ident *loc, int32_t global_tid,
+                                   int32_t sched, int64_t lower, int64_t upper,
+                                   int64_t incr, int64_t chunk);
+EXTERN void __kmpc_dispatch_init_8u(kmp_Ident *loc, int32_t global_tid,
+                                    int32_t sched, uint64_t lower,
+                                    uint64_t upper, int64_t incr,
+                                    int64_t chunk);
+
+EXTERN int __kmpc_dispatch_next_4(kmp_Ident *loc, int32_t global_tid,
+                                  int32_t *plastiter, int32_t *plower,
+                                  int32_t *pupper, int32_t *pstride);
+EXTERN int __kmpc_dispatch_next_4u(kmp_Ident *loc, int32_t global_tid,
+                                   int32_t *plastiter, uint32_t *plower,
+                                   uint32_t *pupper, int32_t *pstride);
+EXTERN int __kmpc_dispatch_next_8(kmp_Ident *loc, int32_t global_tid,
+                                  int32_t *plastiter, int64_t *plower,
+                                  int64_t *pupper, int64_t *pstride);
+EXTERN int __kmpc_dispatch_next_8u(kmp_Ident *loc, int32_t global_tid,
+                                   int32_t *plastiter, uint64_t *plower,
+                                   uint64_t *pupper, int64_t *pstride);
+
+EXTERN void __kmpc_dispatch_fini_4(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_dispatch_fini_4u(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_dispatch_fini_8(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_dispatch_fini_8u(kmp_Ident *loc, int32_t global_tid);
+
+// Support for reducing conditional lastprivate variables
+EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc,
+                                                  int32_t global_tid,
+                                                  int32_t varNum, void *array);
+
+// reduction
+EXTERN void __kmpc_nvptx_end_reduce(int32_t global_tid);
+EXTERN void __kmpc_nvptx_end_reduce_nowait(int32_t global_tid);
+EXTERN __attribute__((deprecated)) int32_t __kmpc_nvptx_parallel_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
+EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_v2(
+    kmp_Ident *loc, int32_t global_tid, int32_t num_vars, size_t reduce_size,
+    void *reduce_data, kmp_ShuffleReductFctPtr shflFct,
+    kmp_InterWarpCopyFctPtr cpyFct);
+EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_spmd(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
+EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_generic(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
+EXTERN int32_t __kmpc_nvptx_simd_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct);
+EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
+EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple_spmd(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
+EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple_generic(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr sratchFct, kmp_LoadReduceFctPtr ldFct);
+EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple(kmp_Ident *loc,
+                                                       int32_t global_tid,
+                                                       kmp_CriticalName *crit);
+EXTERN void __kmpc_nvptx_teams_end_reduce_nowait_simple(kmp_Ident *loc,
+                                                        int32_t global_tid,
+                                                        kmp_CriticalName *crit);
+EXTERN int32_t __kmpc_shuffle_int32(int32_t val, int16_t delta, int16_t size);
+EXTERN int64_t __kmpc_shuffle_int64(int64_t val, int16_t delta, int16_t size);
+
+// sync barrier
+EXTERN void __kmpc_barrier(kmp_Ident *loc_ref, int32_t tid);
+EXTERN void __kmpc_barrier_simple_spmd(kmp_Ident *loc_ref, int32_t tid);
+EXTERN void __kmpc_barrier_simple_generic(kmp_Ident *loc_ref, int32_t tid);
+EXTERN int32_t __kmpc_cancel_barrier(kmp_Ident *loc, int32_t global_tid);
+
+// single
+EXTERN int32_t __kmpc_single(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_end_single(kmp_Ident *loc, int32_t global_tid);
+
+// sync
+EXTERN int32_t __kmpc_master(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_end_master(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_ordered(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_end_ordered(kmp_Ident *loc, int32_t global_tid);
+EXTERN void __kmpc_critical(kmp_Ident *loc, int32_t global_tid,
+                            kmp_CriticalName *crit);
+EXTERN void __kmpc_end_critical(kmp_Ident *loc, int32_t global_tid,
+                                kmp_CriticalName *crit);
+EXTERN void __kmpc_flush(kmp_Ident *loc);
+
+// vote
+EXTERN int32_t __kmpc_warp_active_thread_mask();
+
+// tasks
+EXTERN kmp_TaskDescr *__kmpc_omp_task_alloc(kmp_Ident *loc,
+                                            uint32_t global_tid, int32_t flag,
+                                            size_t sizeOfTaskInclPrivate,
+                                            size_t sizeOfSharedTable,
+                                            kmp_TaskFctPtr sub);
+EXTERN int32_t __kmpc_omp_task(kmp_Ident *loc, uint32_t global_tid,
+                               kmp_TaskDescr *newLegacyTaskDescr);
+EXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid,
+                                         kmp_TaskDescr *newLegacyTaskDescr,
+                                         int32_t depNum, void *depList,
+                                         int32_t noAliasDepNum,
+                                         void *noAliasDepList);
+EXTERN void __kmpc_omp_task_begin_if0(kmp_Ident *loc, uint32_t global_tid,
+                                      kmp_TaskDescr *newLegacyTaskDescr);
+EXTERN void __kmpc_omp_task_complete_if0(kmp_Ident *loc, uint32_t global_tid,
+                                         kmp_TaskDescr *newLegacyTaskDescr);
+EXTERN void __kmpc_omp_wait_deps(kmp_Ident *loc, uint32_t global_tid,
+                                 int32_t depNum, void *depList,
+                                 int32_t noAliasDepNum, void *noAliasDepList);
+EXTERN void __kmpc_taskgroup(kmp_Ident *loc, uint32_t global_tid);
+EXTERN void __kmpc_end_taskgroup(kmp_Ident *loc, uint32_t global_tid);
+EXTERN int32_t __kmpc_omp_taskyield(kmp_Ident *loc, uint32_t global_tid,
+                                    int end_part);
+EXTERN int32_t __kmpc_omp_taskwait(kmp_Ident *loc, uint32_t global_tid);
+EXTERN void __kmpc_taskloop(kmp_Ident *loc, uint32_t global_tid,
+                            kmp_TaskDescr *newKmpTaskDescr, int if_val,
+                            uint64_t *lb, uint64_t *ub, int64_t st, int nogroup,
+                            int32_t sched, uint64_t grainsize, void *task_dup);
+
+// cancel
+EXTERN int32_t __kmpc_cancellationpoint(kmp_Ident *loc, int32_t global_tid,
+                                        int32_t cancelVal);
+EXTERN int32_t __kmpc_cancel(kmp_Ident *loc, int32_t global_tid,
+                             int32_t cancelVal);
+
+// non standard
+EXTERN void __kmpc_kernel_init_params(void *ReductionScratchpadPtr);
+EXTERN void __kmpc_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime);
+EXTERN void __kmpc_kernel_deinit(int16_t IsOMPRuntimeInitialized);
+EXTERN void __kmpc_spmd_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime,
+                                    int16_t RequiresDataSharing);
+EXTERN __attribute__((deprecated)) void __kmpc_spmd_kernel_deinit();
+EXTERN void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime);
+EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
+                                           int16_t IsOMPRuntimeInitialized);
+EXTERN bool __kmpc_kernel_parallel(void **WorkFn,
+                                   int16_t IsOMPRuntimeInitialized);
+EXTERN void __kmpc_kernel_end_parallel();
+EXTERN bool __kmpc_kernel_convergent_parallel(void *buffer, uint32_t Mask,
+                                              bool *IsFinal,
+                                              int32_t *LaneSource);
+EXTERN void __kmpc_kernel_end_convergent_parallel(void *buffer);
+EXTERN bool __kmpc_kernel_convergent_simd(void *buffer, uint32_t Mask,
+                                          bool *IsFinal, int32_t *LaneSource,
+                                          int32_t *LaneId, int32_t *NumLanes);
+EXTERN void __kmpc_kernel_end_convergent_simd(void *buffer);
+
+
+EXTERN void __kmpc_data_sharing_init_stack();
+EXTERN void __kmpc_data_sharing_init_stack_spmd();
+EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t size,
+    int16_t UseSharedMemory);
+EXTERN void *__kmpc_data_sharing_push_stack(size_t size, int16_t UseSharedMemory);
+EXTERN void __kmpc_data_sharing_pop_stack(void *a);
+EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs);
+EXTERN void __kmpc_end_sharing_variables();
+EXTERN void __kmpc_get_shared_variables(void ***GlobalArgs);
+
+// The slot used for data sharing by the master and worker threads. We use a
+// complete (default size version and an incomplete one so that we allow sizes
+// greater than the default).
+struct __kmpc_data_sharing_slot {
+  __kmpc_data_sharing_slot *Next;
+  __kmpc_data_sharing_slot *Prev;
+  void *PrevSlotStackPtr;
+  void *DataEnd;
+  char Data[];
+};
+EXTERN void
+__kmpc_initialize_data_sharing_environment(__kmpc_data_sharing_slot *RootS,
+                                           size_t InitialDataSize);
+EXTERN void *__kmpc_data_sharing_environment_begin(
+    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
+    void **SavedSharedFrame, int32_t *SavedActiveThreads,
+    size_t SharingDataSize, size_t SharingDefaultDataSize,
+    int16_t IsOMPRuntimeInitialized);
+EXTERN void __kmpc_data_sharing_environment_end(
+    __kmpc_data_sharing_slot **SavedSharedSlot, void **SavedSharedStack,
+    void **SavedSharedFrame, int32_t *SavedActiveThreads, int32_t IsEntryPoint);
+
+EXTERN void *
+__kmpc_get_data_sharing_environment_frame(int32_t SourceThreadID,
+                                          int16_t IsOMPRuntimeInitialized);
+
+// SPMD execution mode interrogation function.
+EXTERN int8_t __kmpc_is_spmd_exec_mode();
+
+EXTERN void __kmpc_get_team_static_memory(int16_t isSPMDExecutionMode,
+                                          const void *buf, size_t size,
+                                          int16_t is_shared, const void **res);
+
+EXTERN void __kmpc_restore_team_static_memory(int16_t isSPMDExecutionMode,
+                                              int16_t is_shared);
+
+#endif

libomptarget/deviceRTLs/nvptx/src/libcall.cu

@@ -0,0 +1,515 @@
+//===------------ libcall.cu - NVPTX OpenMP user calls ----------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the OpenMP runtime functions that can be
+// invoked by the user in an OpenMP region
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+// Timer precision is 1ns
+#define TIMER_PRECISION ((double)1E-9)
+
+EXTERN double omp_get_wtick(void) {
+  PRINT(LD_IO, "omp_get_wtick() returns %g\n", TIMER_PRECISION);
+  return TIMER_PRECISION;
+}
+
+EXTERN double omp_get_wtime(void) {
+  unsigned long long nsecs;
+  asm("mov.u64  %0, %%globaltimer;" : "=l"(nsecs));
+  double rc = (double)nsecs * TIMER_PRECISION;
+  PRINT(LD_IO, "call omp_get_wtime() returns %g\n", rc);
+  return rc;
+}
+
+EXTERN void omp_set_num_threads(int num) {
+  // Ignore it for SPMD mode.
+  if (isSPMDMode())
+    return;
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
+  PRINT(LD_IO, "call omp_set_num_threads(num %d)\n", num);
+  if (num <= 0) {
+    WARNING0(LW_INPUT, "expected positive num; ignore\n");
+  } else {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        getMyTopTaskDescriptor(/*isSPMDExecutionMode=*/false);
+    currTaskDescr->NThreads() = num;
+  }
+}
+
+EXTERN int omp_get_num_threads(void) {
+  bool isSPMDExecutionMode = isSPMDMode();
+  int tid = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
+  int rc =
+      GetNumberOfOmpThreads(tid, isSPMDExecutionMode, isRuntimeUninitialized());
+  PRINT(LD_IO, "call omp_get_num_threads() return %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_max_threads(void) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    // We're already in parallel region.
+    return 1;  // default is 1 thread avail
+  }
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      getMyTopTaskDescriptor(isSPMDMode());
+  int rc = 1; // default is 1 thread avail
+  if (!currTaskDescr->InParallelRegion()) {
+    // Not currently in a parallel region, return what was set.
+    rc = currTaskDescr->NThreads();
+    ASSERT0(LT_FUSSY, rc >= 0, "bad number of threads");
+  }
+  PRINT(LD_IO, "call omp_get_max_threads() return %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_thread_limit(void) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return 0;  // default is 0
+  }
+  // per contention group.. meaning threads in current team
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      getMyTopTaskDescriptor(isSPMDMode());
+  int rc = currTaskDescr->ThreadLimit();
+  PRINT(LD_IO, "call omp_get_thread_limit() return %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_thread_num() {
+  bool isSPMDExecutionMode = isSPMDMode();
+  int tid = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
+  int rc = GetOmpThreadId(tid, isSPMDExecutionMode, isRuntimeUninitialized());
+  PRINT(LD_IO, "call omp_get_thread_num() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_num_procs(void) {
+  int rc = GetNumberOfProcsInDevice(isSPMDMode());
+  PRINT(LD_IO, "call omp_get_num_procs() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_in_parallel(void) {
+  int rc = 0;
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    rc = 1;  // SPMD mode is always in parallel.
+  } else {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        getMyTopTaskDescriptor(isSPMDMode());
+    if (currTaskDescr->InParallelRegion()) {
+      rc = 1;
+    }
+  }
+  PRINT(LD_IO, "call omp_in_parallel() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_in_final(void) {
+  // treat all tasks as final... Specs may expect runtime to keep
+  // track more precisely if a task was actively set by users... This
+  // is not explicitely specified; will treat as if runtime can
+  // actively decide to put a non-final task into a final one.
+  int rc = 1;
+  PRINT(LD_IO, "call omp_in_final() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN void omp_set_dynamic(int flag) {
+  PRINT(LD_IO, "call omp_set_dynamic(%d) is ignored (no support)\n", flag);
+}
+
+EXTERN int omp_get_dynamic(void) {
+  int rc = 0;
+  PRINT(LD_IO, "call omp_get_dynamic() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN void omp_set_nested(int flag) {
+  PRINT(LD_IO, "call omp_set_nested(%d) is ignored (no nested support)\n",
+        flag);
+}
+
+EXTERN int omp_get_nested(void) {
+  int rc = 0;
+  PRINT(LD_IO, "call omp_get_nested() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN void omp_set_max_active_levels(int level) {
+  PRINT(LD_IO,
+        "call omp_set_max_active_levels(%d) is ignored (no nested support)\n",
+        level);
+}
+
+EXTERN int omp_get_max_active_levels(void) {
+  int rc = 1;
+  PRINT(LD_IO, "call omp_get_max_active_levels() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_level(void) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return parallelLevel;
+  }
+  int level = 0;
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      getMyTopTaskDescriptor(isSPMDMode());
+  ASSERT0(LT_FUSSY, currTaskDescr,
+          "do not expect fct to be called in a non-active thread");
+  do {
+    if (currTaskDescr->IsParallelConstruct()) {
+      level++;
+    }
+    currTaskDescr = currTaskDescr->GetPrevTaskDescr();
+  } while (currTaskDescr);
+  PRINT(LD_IO, "call omp_get_level() returns %d\n", level);
+  return level;
+}
+
+EXTERN int omp_get_active_level(void) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return 1;
+  }
+  int level = 0; // no active level parallelism
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      getMyTopTaskDescriptor(isSPMDMode());
+  ASSERT0(LT_FUSSY, currTaskDescr,
+          "do not expect fct to be called in a non-active thread");
+  do {
+    if (currTaskDescr->ThreadsInTeam() > 1) {
+      // has a parallel with more than one thread in team
+      level = 1;
+      break;
+    }
+    currTaskDescr = currTaskDescr->GetPrevTaskDescr();
+  } while (currTaskDescr);
+  PRINT(LD_IO, "call omp_get_active_level() returns %d\n", level)
+  return level;
+}
+
+EXTERN int omp_get_ancestor_thread_num(int level) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return level == 1 ? GetThreadIdInBlock() : 0;
+  }
+  int rc = -1;
+  if (level == 0) {
+    rc = 0;
+  } else if (level > 0) {
+    int totLevel = omp_get_level();
+    if (level <= totLevel) {
+      omptarget_nvptx_TaskDescr *currTaskDescr =
+          getMyTopTaskDescriptor(isSPMDMode());
+      int steps = totLevel - level;
+      PRINT(LD_IO, "backtrack %d steps\n", steps);
+      ASSERT0(LT_FUSSY, currTaskDescr,
+              "do not expect fct to be called in a non-active thread");
+      do {
+        if (DON(LD_IOD)) {
+          // print current state
+          omp_sched_t sched = currTaskDescr->GetRuntimeSched();
+          PRINT(LD_ALL,
+                "task descr %s %d: %s, in par %d, rt sched %d,"
+                " chunk %" PRIu64 "; tid %d, tnum %d, nthreads %d\n",
+                "ancestor", steps,
+                (currTaskDescr->IsParallelConstruct() ? "par" : "task"),
+                (int)currTaskDescr->InParallelRegion(), (int)sched,
+                currTaskDescr->RuntimeChunkSize(),
+                (int)currTaskDescr->ThreadId(),
+                (int)currTaskDescr->ThreadsInTeam(),
+                (int)currTaskDescr->NThreads());
+        }
+
+        if (currTaskDescr->IsParallelConstruct()) {
+          // found the level
+          if (!steps) {
+            rc = currTaskDescr->ThreadId();
+            break;
+          }
+          steps--;
+        }
+        currTaskDescr = currTaskDescr->GetPrevTaskDescr();
+      } while (currTaskDescr);
+      ASSERT0(LT_FUSSY, !steps, "expected to find all steps");
+    }
+  }
+  PRINT(LD_IO, "call omp_get_ancestor_thread_num(level %d) returns %d\n", level,
+        rc)
+  return rc;
+}
+
+EXTERN int omp_get_team_size(int level) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return level == 1 ? GetNumberOfThreadsInBlock() : 1;
+  }
+  int rc = -1;
+  if (level == 0) {
+    rc = 1;
+  } else if (level > 0) {
+    int totLevel = omp_get_level();
+    if (level <= totLevel) {
+      omptarget_nvptx_TaskDescr *currTaskDescr =
+          getMyTopTaskDescriptor(isSPMDMode());
+      int steps = totLevel - level;
+      ASSERT0(LT_FUSSY, currTaskDescr,
+              "do not expect fct to be called in a non-active thread");
+      do {
+        if (currTaskDescr->IsParallelConstruct()) {
+          if (!steps) {
+            // found the level
+            rc = currTaskDescr->ThreadsInTeam();
+            break;
+          }
+          steps--;
+        }
+        currTaskDescr = currTaskDescr->GetPrevTaskDescr();
+      } while (currTaskDescr);
+      ASSERT0(LT_FUSSY, !steps, "expected to find all steps");
+    }
+  }
+  PRINT(LD_IO, "call omp_get_team_size(level %d) returns %d\n", level, rc)
+  return rc;
+}
+
+EXTERN void omp_get_schedule(omp_sched_t *kind, int *modifier) {
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    *kind = omp_sched_static;
+    *modifier = 1;
+  } else {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        getMyTopTaskDescriptor(isSPMDMode());
+    *kind = currTaskDescr->GetRuntimeSched();
+    *modifier = currTaskDescr->RuntimeChunkSize();
+  }
+  PRINT(LD_IO, "call omp_get_schedule returns sched %d and modif %d\n",
+        (int)*kind, *modifier);
+}
+
+EXTERN void omp_set_schedule(omp_sched_t kind, int modifier) {
+  PRINT(LD_IO, "call omp_set_schedule(sched %d, modif %d)\n", (int)kind,
+        modifier);
+  if (isRuntimeUninitialized()) {
+    ASSERT0(LT_FUSSY, isSPMDMode(),
+            "Expected SPMD mode only with uninitialized runtime.");
+    return;
+  }
+  if (kind >= omp_sched_static && kind < omp_sched_auto) {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        getMyTopTaskDescriptor(isSPMDMode());
+    currTaskDescr->SetRuntimeSched(kind);
+    currTaskDescr->RuntimeChunkSize() = modifier;
+    PRINT(LD_IOD, "omp_set_schedule did set sched %d & modif %" PRIu64 "\n",
+          (int)currTaskDescr->GetRuntimeSched(),
+          currTaskDescr->RuntimeChunkSize());
+  }
+}
+
+EXTERN omp_proc_bind_t omp_get_proc_bind(void) {
+  PRINT0(LD_IO, "call omp_get_proc_bin() is true, regardless on state\n");
+  return omp_proc_bind_true;
+}
+
+EXTERN int omp_get_num_places(void) {
+  PRINT0(LD_IO, "call omp_get_num_places() returns 0\n");
+  return 0;
+}
+
+EXTERN int omp_get_place_num_procs(int place_num) {
+  PRINT0(LD_IO, "call omp_get_place_num_procs() returns 0\n");
+  return 0;
+}
+
+EXTERN void omp_get_place_proc_ids(int place_num, int *ids) {
+  PRINT0(LD_IO, "call to omp_get_place_proc_ids()\n");
+}
+
+EXTERN int omp_get_place_num(void) {
+  PRINT0(LD_IO, "call to omp_get_place_num() returns 0\n");
+  return 0;
+}
+
+EXTERN int omp_get_partition_num_places(void) {
+  PRINT0(LD_IO, "call to omp_get_partition_num_places() returns 0\n");
+  return 0;
+}
+
+EXTERN void omp_get_partition_place_nums(int *place_nums) {
+  PRINT0(LD_IO, "call to omp_get_partition_place_nums()\n");
+}
+
+EXTERN int omp_get_cancellation(void) {
+  int rc = FALSE; // currently false only
+  PRINT(LD_IO, "call omp_get_cancellation() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN void omp_set_default_device(int deviceId) {
+  PRINT0(LD_IO, "call omp_get_default_device() is undef on device\n");
+}
+
+EXTERN int omp_get_default_device(void) {
+  PRINT0(LD_IO,
+         "call omp_get_default_device() is undef on device, returns 0\n");
+  return 0;
+}
+
+EXTERN int omp_get_num_devices(void) {
+  PRINT0(LD_IO, "call omp_get_num_devices() is undef on device, returns 0\n");
+  return 0;
+}
+
+EXTERN int omp_get_num_teams(void) {
+  int rc = GetNumberOfOmpTeams();
+  PRINT(LD_IO, "call omp_get_num_teams() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_get_team_num() {
+  int rc = GetOmpTeamId();
+  PRINT(LD_IO, "call omp_get_team_num() returns %d\n", rc);
+  return rc;
+}
+
+EXTERN int omp_is_initial_device(void) {
+  PRINT0(LD_IO, "call omp_is_initial_device() returns 0\n");
+  return 0; // 0 by def on device
+}
+
+// Unspecified on the device.
+EXTERN int omp_get_initial_device(void) {
+  PRINT0(LD_IO, "call omp_get_initial_device() returns 0\n");
+  return 0;
+}
+
+// Unused for now.
+EXTERN int omp_get_max_task_priority(void) {
+  PRINT0(LD_IO, "call omp_get_max_task_priority() returns 0\n");
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// locks
+////////////////////////////////////////////////////////////////////////////////
+
+#define __OMP_SPIN 1000
+#define UNSET 0
+#define SET 1
+
+EXTERN void omp_init_lock(omp_lock_t *lock) {
+  omp_unset_lock(lock);
+  PRINT0(LD_IO, "call omp_init_lock()\n");
+}
+
+EXTERN void omp_destroy_lock(omp_lock_t *lock) {
+  omp_unset_lock(lock);
+  PRINT0(LD_IO, "call omp_destroy_lock()\n");
+}
+
+EXTERN void omp_set_lock(omp_lock_t *lock) {
+  // int atomicCAS(int* address, int compare, int val);
+  // (old == compare ? val : old)
+
+  // TODO: not sure spinning is a good idea here..
+  while (atomicCAS(lock, UNSET, SET) != UNSET) {
+    clock_t start = clock();
+    clock_t now;
+    for (;;) {
+      now = clock();
+      clock_t cycles = now > start ? now - start : now + (0xffffffff - start);
+      if (cycles >= __OMP_SPIN * blockIdx.x) {
+        break;
+      }
+    }
+  } // wait for 0 to be the read value
+
+  PRINT0(LD_IO, "call omp_set_lock()\n");
+}
+
+EXTERN void omp_unset_lock(omp_lock_t *lock) {
+  (void)atomicExch(lock, UNSET);
+
+  PRINT0(LD_IO, "call omp_unset_lock()\n");
+}
+
+EXTERN int omp_test_lock(omp_lock_t *lock) {
+  // int atomicCAS(int* address, int compare, int val);
+  // (old == compare ? val : old)
+  int ret = atomicAdd(lock, 0);
+
+  PRINT(LD_IO, "call omp_test_lock() return %d\n", ret);
+
+  return ret;
+}
+
+// for xlf Fotran
+// Fotran, the return is LOGICAL type
+
+#define FLOGICAL long
+EXTERN FLOGICAL __xlf_omp_is_initial_device_i8() {
+  int ret = omp_is_initial_device();
+  if (ret == 0)
+    return (FLOGICAL)0;
+  else
+    return (FLOGICAL)1;
+}
+
+EXTERN int __xlf_omp_is_initial_device_i4() {
+  int ret = omp_is_initial_device();
+  if (ret == 0)
+    return 0;
+  else
+    return 1;
+}
+
+EXTERN long __xlf_omp_get_team_num_i4() {
+  int ret = omp_get_team_num();
+  return (long)ret;
+}
+
+EXTERN long __xlf_omp_get_num_teams_i4() {
+  int ret = omp_get_num_teams();
+  return (long)ret;
+}
+
+EXTERN void xlf_debug_print_int(int *p) {
+  printf("xlf DEBUG %d): %p %d\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
+}
+
+EXTERN void xlf_debug_print_long(long *p) {
+  printf("xlf DEBUG %d): %p %ld\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
+}
+
+EXTERN void xlf_debug_print_float(float *p) {
+  printf("xlf DEBUG %d): %p %f\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
+}
+
+EXTERN void xlf_debug_print_double(double *p) {
+  printf("xlf DEBUG %d): %p %f\n", omp_get_team_num(), p, p == 0 ? 0 : *p);
+}
+
+EXTERN void xlf_debug_print_addr(void *p) {
+  printf("xlf DEBUG %d): %p \n", omp_get_team_num(), p);
+}

libomptarget/deviceRTLs/nvptx/src/loop.cu

@@ -0,0 +1,806 @@
+//===------------ loop.cu - NVPTX OpenMP loop constructs --------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of the KMPC interface
+// for the loop construct plus other worksharing constructs that use the same
+// interface as loops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+// template class that encapsulate all the helper functions
+//
+// T is loop iteration type (32 | 64)  (unsigned | signed)
+// ST is the signed version of T
+////////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////////
+
+template <typename T, typename ST> class omptarget_nvptx_LoopSupport {
+public:
+  ////////////////////////////////////////////////////////////////////////////////
+  // Loop with static scheduling with chunk
+
+  // Generic implementation of OMP loop scheduling with static policy
+  /*! \brief Calculate initial bounds for static loop and stride
+   *  @param[in] loc location in code of the call (not used here)
+   *  @param[in] global_tid global thread id
+   *  @param[in] schetype type of scheduling (see omptarget-nvptx.h)
+   *  @param[in] plastiter pointer to last iteration
+   *  @param[in,out] pointer to loop lower bound. it will contain value of
+   *  lower bound of first chunk
+   *  @param[in,out] pointer to loop upper bound. It will contain value of
+   *  upper bound of first chunk
+   *  @param[in,out] pointer to loop stride. It will contain value of stride
+   *  between two successive chunks executed by the same thread
+   *  @param[in] loop increment bump
+   *  @param[in] chunk size
+   */
+
+  // helper function for static chunk
+  INLINE static void ForStaticChunk(int &last, T &lb, T &ub, ST &stride,
+                                    ST chunk, T entityId, T numberOfEntities) {
+    // each thread executes multiple chunks all of the same size, except
+    // the last one
+
+    // distance between two successive chunks
+    stride = numberOfEntities * chunk;
+    lb = lb + entityId * chunk;
+    T inputUb = ub;
+    ub = lb + chunk - 1; // Clang uses i <= ub
+    // Say ub' is the begining of the last chunk. Then who ever has a
+    // lower bound plus a multiple of the increment equal to ub' is
+    // the last one.
+    T beginingLastChunk = inputUb - (inputUb % chunk);
+    last = ((beginingLastChunk - lb) % stride) == 0;
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  // Loop with static scheduling without chunk
+
+  // helper function for static no chunk
+  INLINE static void ForStaticNoChunk(int &last, T &lb, T &ub, ST &stride,
+                                      ST &chunk, T entityId,
+                                      T numberOfEntities) {
+    // No chunk size specified.  Each thread or warp gets at most one
+    // chunk; chunks are all almost of equal size
+    T loopSize = ub - lb + 1;
+
+    chunk = loopSize / numberOfEntities;
+    T leftOver = loopSize - chunk * numberOfEntities;
+
+    if (entityId < leftOver) {
+      chunk++;
+      lb = lb + entityId * chunk;
+    } else {
+      lb = lb + entityId * chunk + leftOver;
+    }
+
+    T inputUb = ub;
+    ub = lb + chunk - 1; // Clang uses i <= ub
+    last = lb <= inputUb && inputUb <= ub;
+    stride = loopSize; // make sure we only do 1 chunk per warp
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  // Support for Static Init
+
+  INLINE static void for_static_init(int32_t gtid, int32_t schedtype,
+                                     int32_t *plastiter, T *plower, T *pupper,
+                                     ST *pstride, ST chunk,
+                                     bool IsSPMDExecutionMode,
+                                     bool IsRuntimeUninitialized) {
+    // When IsRuntimeUninitialized is true, we assume that the caller is
+    // in an L0 parallel region and that all worker threads participate.
+
+    int tid = GetLogicalThreadIdInBlock(IsSPMDExecutionMode);
+
+    // Assume we are in teams region or that we use a single block
+    // per target region
+    ST numberOfActiveOMPThreads = GetNumberOfOmpThreads(
+        tid, IsSPMDExecutionMode, IsRuntimeUninitialized);
+
+    // All warps that are in excess of the maximum requested, do
+    // not execute the loop
+    PRINT(LD_LOOP,
+          "OMP Thread %d: schedule type %d, chunk size = %lld, mytid "
+          "%d, num tids %d\n",
+          (int)gtid, (int)schedtype, (long long)chunk, (int)gtid,
+          (int)numberOfActiveOMPThreads);
+    ASSERT0(LT_FUSSY, gtid < numberOfActiveOMPThreads,
+            "current thread is not needed here; error");
+
+    // copy
+    int lastiter = 0;
+    T lb = *plower;
+    T ub = *pupper;
+    ST stride = *pstride;
+    // init
+    switch (SCHEDULE_WITHOUT_MODIFIERS(schedtype)) {
+    case kmp_sched_static_chunk: {
+      if (chunk > 0) {
+        ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
+                       numberOfActiveOMPThreads);
+        break;
+      }
+    } // note: if chunk <=0, use nochunk
+    case kmp_sched_static_balanced_chunk: {
+      if (chunk > 0) {
+        // round up to make sure the chunk is enough to cover all iterations
+        T tripCount = ub - lb + 1; // +1 because ub is inclusive
+        T span = (tripCount + numberOfActiveOMPThreads - 1) /
+                 numberOfActiveOMPThreads;
+        // perform chunk adjustment
+        chunk = (span + chunk - 1) & ~(chunk - 1);
+
+        ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb.");
+        T oldUb = ub;
+        ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
+                       numberOfActiveOMPThreads);
+        if (ub > oldUb)
+          ub = oldUb;
+        break;
+      }
+    } // note: if chunk <=0, use nochunk
+    case kmp_sched_static_nochunk: {
+      ForStaticNoChunk(lastiter, lb, ub, stride, chunk, gtid,
+                       numberOfActiveOMPThreads);
+      break;
+    }
+    case kmp_sched_distr_static_chunk: {
+      if (chunk > 0) {
+        ForStaticChunk(lastiter, lb, ub, stride, chunk, GetOmpTeamId(),
+                       GetNumberOfOmpTeams());
+        break;
+      } // note: if chunk <=0, use nochunk
+    }
+    case kmp_sched_distr_static_nochunk: {
+      ForStaticNoChunk(lastiter, lb, ub, stride, chunk, GetOmpTeamId(),
+                       GetNumberOfOmpTeams());
+      break;
+    }
+    case kmp_sched_distr_static_chunk_sched_static_chunkone: {
+      ForStaticChunk(lastiter, lb, ub, stride, chunk,
+                     numberOfActiveOMPThreads * GetOmpTeamId() + gtid,
+                     GetNumberOfOmpTeams() * numberOfActiveOMPThreads);
+      break;
+    }
+    default: {
+      ASSERT(LT_FUSSY, FALSE, "unknown schedtype %d", (int)schedtype);
+      PRINT(LD_LOOP, "unknown schedtype %d, revert back to static chunk\n",
+            (int)schedtype);
+      ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
+                     numberOfActiveOMPThreads);
+      break;
+    }
+    }
+    // copy back
+    *plastiter = lastiter;
+    *plower = lb;
+    *pupper = ub;
+    *pstride = stride;
+    PRINT(LD_LOOP,
+          "Got sched: Active %d, total %d: lb %lld, ub %lld, stride %lld, last "
+          "%d\n",
+          (int)numberOfActiveOMPThreads, (int)GetNumberOfWorkersInTeam(),
+          (long long)(*plower), (long long)(*pupper), (long long)(*pstride),
+          (int)lastiter);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  // Support for dispatch Init
+
+  INLINE static int OrderedSchedule(kmp_sched_t schedule) {
+    return schedule >= kmp_sched_ordered_first &&
+           schedule <= kmp_sched_ordered_last;
+  }
+
+  INLINE static void dispatch_init(kmp_Ident *loc, int32_t threadId,
+                                   kmp_sched_t schedule, T lb, T ub, ST st,
+                                   ST chunk) {
+    ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+            "Expected non-SPMD mode + initialized runtime.");
+    int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+    omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(tid);
+    T tnum = currTaskDescr->ThreadsInTeam();
+    T tripCount = ub - lb + 1; // +1 because ub is inclusive
+    ASSERT0(LT_FUSSY, threadId < tnum,
+            "current thread is not needed here; error");
+
+    /* Currently just ignore the monotonic and non-monotonic modifiers
+     * (the compiler isn't producing them * yet anyway).
+     * When it is we'll want to look at them somewhere here and use that
+     * information to add to our schedule choice. We shouldn't need to pass
+     * them on, they merely affect which schedule we can legally choose for
+     * various dynamic cases. (In paritcular, whether or not a stealing scheme
+     * is legal).
+     */
+    schedule = SCHEDULE_WITHOUT_MODIFIERS(schedule);
+
+    // Process schedule.
+    if (tnum == 1 || tripCount <= 1 || OrderedSchedule(schedule)) {
+      if (OrderedSchedule(schedule))
+        __kmpc_barrier(loc, threadId);
+      PRINT(LD_LOOP,
+            "go sequential as tnum=%ld, trip count %lld, ordered sched=%d\n",
+            (long)tnum, (long long)tripCount, (int)schedule);
+      schedule = kmp_sched_static_chunk;
+      chunk = tripCount; // one thread gets the whole loop
+    } else if (schedule == kmp_sched_runtime) {
+      // process runtime
+      omp_sched_t rtSched = currTaskDescr->GetRuntimeSched();
+      chunk = currTaskDescr->RuntimeChunkSize();
+      switch (rtSched) {
+      case omp_sched_static: {
+        if (chunk > 0)
+          schedule = kmp_sched_static_chunk;
+        else
+          schedule = kmp_sched_static_nochunk;
+        break;
+      }
+      case omp_sched_auto: {
+        schedule = kmp_sched_static_chunk;
+        chunk = 1;
+        break;
+      }
+      case omp_sched_dynamic:
+      case omp_sched_guided: {
+        schedule = kmp_sched_dynamic;
+        break;
+      }
+      }
+      PRINT(LD_LOOP, "Runtime sched is %d with chunk %lld\n", (int)schedule,
+            (long long)chunk);
+    } else if (schedule == kmp_sched_auto) {
+      schedule = kmp_sched_static_chunk;
+      chunk = 1;
+      PRINT(LD_LOOP, "Auto sched is %d with chunk %lld\n", (int)schedule,
+            (long long)chunk);
+    } else {
+      PRINT(LD_LOOP, "Dyn sched is %d with chunk %lld\n", (int)schedule,
+            (long long)chunk);
+      ASSERT(LT_FUSSY,
+             schedule == kmp_sched_dynamic || schedule == kmp_sched_guided,
+             "unknown schedule %d & chunk %lld\n", (int)schedule,
+             (long long)chunk);
+    }
+
+    // init schedules
+    if (schedule == kmp_sched_static_chunk) {
+      ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value");
+      // save sched state
+      omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
+      // save ub
+      omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
+      // compute static chunk
+      ST stride;
+      int lastiter = 0;
+      ForStaticChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
+      // save computed params
+      omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
+      omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
+      PRINT(LD_LOOP,
+            "dispatch init (static chunk) : num threads = %d, ub =  %" PRId64
+            ", next lower bound = %llu, stride = %llu\n",
+            (int)tnum,
+            omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
+            (unsigned long long)
+                omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
+            (unsigned long long)omptarget_nvptx_threadPrivateContext->Stride(
+                tid));
+    } else if (schedule == kmp_sched_static_balanced_chunk) {
+      ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value");
+      // save sched state
+      omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
+      // save ub
+      omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
+      // compute static chunk
+      ST stride;
+      int lastiter = 0;
+      // round up to make sure the chunk is enough to cover all iterations
+      T span = (tripCount + tnum - 1) / tnum;
+      // perform chunk adjustment
+      chunk = (span + chunk - 1) & ~(chunk - 1);
+
+      T oldUb = ub;
+      ForStaticChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
+      ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb.");
+      if (ub > oldUb)
+        ub = oldUb;
+      // save computed params
+      omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
+      omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
+      PRINT(LD_LOOP,
+            "dispatch init (static chunk) : num threads = %d, ub =  %" PRId64
+            ", next lower bound = %llu, stride = %llu\n",
+            (int)tnum,
+            omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
+            (unsigned long long)
+                omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
+            (unsigned long long)omptarget_nvptx_threadPrivateContext->Stride(
+                tid));
+    } else if (schedule == kmp_sched_static_nochunk) {
+      ASSERT0(LT_FUSSY, chunk == 0, "bad chunk value");
+      // save sched state
+      omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
+      // save ub
+      omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
+      // compute static chunk
+      ST stride;
+      int lastiter = 0;
+      ForStaticNoChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
+      // save computed params
+      omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
+      omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
+      PRINT(LD_LOOP,
+            "dispatch init (static nochunk) : num threads = %d, ub = %" PRId64
+            ", next lower bound = %llu, stride = %llu\n",
+            (int)tnum,
+            omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
+            (unsigned long long)
+                omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
+            (unsigned long long)omptarget_nvptx_threadPrivateContext->Stride(
+                tid));
+    } else if (schedule == kmp_sched_dynamic || schedule == kmp_sched_guided) {
+      // save data
+      omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
+      if (chunk < 1)
+        chunk = 1;
+      omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
+      omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
+      __kmpc_barrier(loc, threadId);
+      if (tid == 0) {
+        omptarget_nvptx_threadPrivateContext->Cnt() = 0;
+        __threadfence_block();
+      }
+      __kmpc_barrier(loc, threadId);
+      PRINT(LD_LOOP,
+            "dispatch init (dyn) : num threads = %d, lb = %llu, ub = %" PRId64
+            ", chunk %" PRIu64 "\n",
+            (int)tnum,
+            (unsigned long long)
+                omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
+            omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
+            omptarget_nvptx_threadPrivateContext->Chunk(tid));
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  // Support for dispatch next
+
+  INLINE static int64_t Shuffle(unsigned active, int64_t val, int leader) {
+    int lo, hi;
+    asm volatile("mov.b64 {%0,%1}, %2;" : "=r"(lo), "=r"(hi) : "l"(val));
+    hi = __SHFL_SYNC(active, hi, leader);
+    lo = __SHFL_SYNC(active, lo, leader);
+    asm volatile("mov.b64 %0, {%1,%2};" : "=l"(val) : "r"(lo), "r"(hi));
+    return val;
+  }
+
+  INLINE static uint64_t NextIter() {
+    unsigned int active = __ACTIVEMASK();
+    int leader = __ffs(active) - 1;
+    int change = __popc(active);
+    unsigned lane_mask_lt;
+    asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lane_mask_lt));
+    unsigned int rank = __popc(active & lane_mask_lt);
+    uint64_t warp_res;
+    if (rank == 0) {
+      warp_res = atomicAdd(
+          (unsigned long long *)&omptarget_nvptx_threadPrivateContext->Cnt(),
+          change);
+    }
+    warp_res = Shuffle(active, warp_res, leader);
+    return warp_res + rank;
+  }
+
+  INLINE static int DynamicNextChunk(T &lb, T &ub, T chunkSize,
+                                     T loopLowerBound, T loopUpperBound) {
+    T N = NextIter();
+    lb = loopLowerBound + N * chunkSize;
+    ub = lb + chunkSize - 1;  // Clang uses i <= ub
+
+    // 3 result cases:
+    //  a. lb and ub < loopUpperBound --> NOT_FINISHED
+    //  b. lb < loopUpperBound and ub >= loopUpperBound: last chunk -->
+    //  NOT_FINISHED
+    //  c. lb and ub >= loopUpperBound: empty chunk --> FINISHED
+    // a.
+    if (lb <= loopUpperBound && ub < loopUpperBound) {
+      PRINT(LD_LOOPD, "lb %lld, ub %lld, loop ub %lld; not finished\n",
+            (long long)lb, (long long)ub, (long long)loopUpperBound);
+      return NOT_FINISHED;
+    }
+    // b.
+    if (lb <= loopUpperBound) {
+      PRINT(LD_LOOPD, "lb %lld, ub %lld, loop ub %lld; clip to loop ub\n",
+            (long long)lb, (long long)ub, (long long)loopUpperBound);
+      ub = loopUpperBound;
+      return LAST_CHUNK;
+    }
+    // c. if we are here, we are in case 'c'
+    lb = loopUpperBound + 2;
+    ub = loopUpperBound + 1;
+    PRINT(LD_LOOPD, "lb %lld, ub %lld, loop ub %lld; finished\n", (long long)lb,
+          (long long)ub, (long long)loopUpperBound);
+    return FINISHED;
+  }
+
+  INLINE static int dispatch_next(kmp_Ident *loc, int32_t gtid, int32_t *plast,
+                                  T *plower, T *pupper, ST *pstride) {
+    ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+            "Expected non-SPMD mode + initialized runtime.");
+    // ID of a thread in its own warp
+
+    // automatically selects thread or warp ID based on selected implementation
+    int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+    ASSERT0(LT_FUSSY,
+            gtid < GetNumberOfOmpThreads(tid, checkSPMDMode(loc),
+                                         checkRuntimeUninitialized(loc)),
+            "current thread is not needed here; error");
+    // retrieve schedule
+    kmp_sched_t schedule =
+        omptarget_nvptx_threadPrivateContext->ScheduleType(tid);
+
+    // xxx reduce to one
+    if (schedule == kmp_sched_static_chunk ||
+        schedule == kmp_sched_static_nochunk) {
+      T myLb = omptarget_nvptx_threadPrivateContext->NextLowerBound(tid);
+      T ub = omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid);
+      // finished?
+      if (myLb > ub) {
+        PRINT(LD_LOOP, "static loop finished with myLb %lld, ub %lld\n",
+              (long long)myLb, (long long)ub);
+        return DISPATCH_FINISHED;
+      }
+      // not finished, save current bounds
+      ST chunk = omptarget_nvptx_threadPrivateContext->Chunk(tid);
+      *plower = myLb;
+      T myUb = myLb + chunk - 1; // Clang uses i <= ub
+      if (myUb > ub)
+        myUb = ub;
+      *pupper = myUb;
+      *plast = (int32_t)(myUb == ub);
+
+      // increment next lower bound by the stride
+      ST stride = omptarget_nvptx_threadPrivateContext->Stride(tid);
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = myLb + stride;
+      PRINT(LD_LOOP, "static loop continues with myLb %lld, myUb %lld\n",
+            (long long)*plower, (long long)*pupper);
+      return DISPATCH_NOTFINISHED;
+    }
+    ASSERT0(LT_FUSSY,
+            schedule == kmp_sched_dynamic || schedule == kmp_sched_guided,
+            "bad sched");
+    T myLb, myUb;
+    int finished = DynamicNextChunk(
+        myLb, myUb, omptarget_nvptx_threadPrivateContext->Chunk(tid),
+        omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
+        omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid));
+
+    if (finished == FINISHED)
+      return DISPATCH_FINISHED;
+
+    // not finished (either not finished or last chunk)
+    *plast = (int32_t)(finished == LAST_CHUNK);
+    *plower = myLb;
+    *pupper = myUb;
+    *pstride = 1;
+
+    PRINT(
+        LD_LOOP,
+        "Got sched: active %d, total %d: lb %lld, ub %lld, stride = %lld, "
+        "last %d\n",
+        (int)GetNumberOfOmpThreads(tid, isSPMDMode(), isRuntimeUninitialized()),
+        (int)GetNumberOfWorkersInTeam(), (long long)*plower, (long long)*pupper,
+        (long long)*pstride, (int)*plast);
+    return DISPATCH_NOTFINISHED;
+  }
+
+  INLINE static void dispatch_fini() {
+    // nothing
+  }
+
+  ////////////////////////////////////////////////////////////////////////////////
+  // end of template class that encapsulate all the helper functions
+  ////////////////////////////////////////////////////////////////////////////////
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP interface implementation (dyn loops)
+////////////////////////////////////////////////////////////////////////////////
+
+// init
+EXTERN void __kmpc_dispatch_init_4(kmp_Ident *loc, int32_t tid,
+                                   int32_t schedule, int32_t lb, int32_t ub,
+                                   int32_t st, int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_dispatch_init_4\n");
+  omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_init(
+      loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk);
+}
+
+EXTERN void __kmpc_dispatch_init_4u(kmp_Ident *loc, int32_t tid,
+                                    int32_t schedule, uint32_t lb, uint32_t ub,
+                                    int32_t st, int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_dispatch_init_4u\n");
+  omptarget_nvptx_LoopSupport<uint32_t, int32_t>::dispatch_init(
+      loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk);
+}
+
+EXTERN void __kmpc_dispatch_init_8(kmp_Ident *loc, int32_t tid,
+                                   int32_t schedule, int64_t lb, int64_t ub,
+                                   int64_t st, int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_dispatch_init_8\n");
+  omptarget_nvptx_LoopSupport<int64_t, int64_t>::dispatch_init(
+      loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk);
+}
+
+EXTERN void __kmpc_dispatch_init_8u(kmp_Ident *loc, int32_t tid,
+                                    int32_t schedule, uint64_t lb, uint64_t ub,
+                                    int64_t st, int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_dispatch_init_8u\n");
+  omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_init(
+      loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk);
+}
+
+// next
+EXTERN int __kmpc_dispatch_next_4(kmp_Ident *loc, int32_t tid, int32_t *p_last,
+                                  int32_t *p_lb, int32_t *p_ub, int32_t *p_st) {
+  PRINT0(LD_IO, "call kmpc_dispatch_next_4\n");
+  return omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_next(
+      loc, tid, p_last, p_lb, p_ub, p_st);
+}
+
+EXTERN int __kmpc_dispatch_next_4u(kmp_Ident *loc, int32_t tid,
+                                   int32_t *p_last, uint32_t *p_lb,
+                                   uint32_t *p_ub, int32_t *p_st) {
+  PRINT0(LD_IO, "call kmpc_dispatch_next_4u\n");
+  return omptarget_nvptx_LoopSupport<uint32_t, int32_t>::dispatch_next(
+      loc, tid, p_last, p_lb, p_ub, p_st);
+}
+
+EXTERN int __kmpc_dispatch_next_8(kmp_Ident *loc, int32_t tid, int32_t *p_last,
+                                  int64_t *p_lb, int64_t *p_ub, int64_t *p_st) {
+  PRINT0(LD_IO, "call kmpc_dispatch_next_8\n");
+  return omptarget_nvptx_LoopSupport<int64_t, int64_t>::dispatch_next(
+      loc, tid, p_last, p_lb, p_ub, p_st);
+}
+
+EXTERN int __kmpc_dispatch_next_8u(kmp_Ident *loc, int32_t tid,
+                                   int32_t *p_last, uint64_t *p_lb,
+                                   uint64_t *p_ub, int64_t *p_st) {
+  PRINT0(LD_IO, "call kmpc_dispatch_next_8u\n");
+  return omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_next(
+      loc, tid, p_last, p_lb, p_ub, p_st);
+}
+
+// fini
+EXTERN void __kmpc_dispatch_fini_4(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_dispatch_fini_4\n");
+  omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_fini();
+}
+
+EXTERN void __kmpc_dispatch_fini_4u(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_dispatch_fini_4u\n");
+  omptarget_nvptx_LoopSupport<uint32_t, int32_t>::dispatch_fini();
+}
+
+EXTERN void __kmpc_dispatch_fini_8(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_dispatch_fini_8\n");
+  omptarget_nvptx_LoopSupport<int64_t, int64_t>::dispatch_fini();
+}
+
+EXTERN void __kmpc_dispatch_fini_8u(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_dispatch_fini_8u\n");
+  omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_fini();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP interface implementation (static loops)
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN void __kmpc_for_static_init_4(kmp_Ident *loc, int32_t global_tid,
+                                     int32_t schedtype, int32_t *plastiter,
+                                     int32_t *plower, int32_t *pupper,
+                                     int32_t *pstride, int32_t incr,
+                                     int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4\n");
+  omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      checkSPMDMode(loc), checkRuntimeUninitialized(loc));
+}
+
+EXTERN void __kmpc_for_static_init_4u(kmp_Ident *loc, int32_t global_tid,
+                                      int32_t schedtype, int32_t *plastiter,
+                                      uint32_t *plower, uint32_t *pupper,
+                                      int32_t *pstride, int32_t incr,
+                                      int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4u\n");
+  omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      checkSPMDMode(loc), checkRuntimeUninitialized(loc));
+}
+
+EXTERN void __kmpc_for_static_init_8(kmp_Ident *loc, int32_t global_tid,
+                                     int32_t schedtype, int32_t *plastiter,
+                                     int64_t *plower, int64_t *pupper,
+                                     int64_t *pstride, int64_t incr,
+                                     int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8\n");
+  omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      checkSPMDMode(loc), checkRuntimeUninitialized(loc));
+}
+
+EXTERN void __kmpc_for_static_init_8u(kmp_Ident *loc, int32_t global_tid,
+                                      int32_t schedtype, int32_t *plastiter,
+                                      uint64_t *plower, uint64_t *pupper,
+                                      int64_t *pstride, int64_t incr,
+                                      int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8u\n");
+  omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      checkSPMDMode(loc), checkRuntimeUninitialized(loc));
+}
+
+EXTERN
+void __kmpc_for_static_init_4_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                          int32_t schedtype, int32_t *plastiter,
+                                          int32_t *plower, int32_t *pupper,
+                                          int32_t *pstride, int32_t incr,
+                                          int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_spmd\n");
+  omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_4u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                           int32_t schedtype,
+                                           int32_t *plastiter, uint32_t *plower,
+                                           uint32_t *pupper, int32_t *pstride,
+                                           int32_t incr, int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_spmd\n");
+  omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_8_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                          int32_t schedtype, int32_t *plastiter,
+                                          int64_t *plower, int64_t *pupper,
+                                          int64_t *pstride, int64_t incr,
+                                          int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_spmd\n");
+  omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_8u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
+                                           int32_t schedtype,
+                                           int32_t *plastiter, uint64_t *plower,
+                                           uint64_t *pupper, int64_t *pstride,
+                                           int64_t incr, int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_spmd\n");
+  omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_4_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
+    int32_t *plower, int32_t *pupper, int32_t *pstride, int32_t incr,
+    int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_generic\n");
+  omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_4u_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
+    uint32_t *plower, uint32_t *pupper, int32_t *pstride, int32_t incr,
+    int32_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_generic\n");
+  omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_8_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
+    int64_t *plower, int64_t *pupper, int64_t *pstride, int64_t incr,
+    int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_generic\n");
+  omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN
+void __kmpc_for_static_init_8u_simple_generic(
+    kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
+    uint64_t *plower, uint64_t *pupper, int64_t *pstride, int64_t incr,
+    int64_t chunk) {
+  PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_generic\n");
+  omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
+      global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
+      /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
+}
+
+EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid) {
+  PRINT0(LD_IO, "call kmpc_for_static_fini\n");
+}
+
+namespace {
+INLINE void syncWorkersInGenericMode(uint32_t NumThreads) {
+  int NumWarps = ((NumThreads + WARPSIZE - 1) / WARPSIZE);
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  // On Volta and newer architectures we require that all lanes in
+  // a warp (at least, all present for the kernel launch) participate in the
+  // barrier.  This is enforced when launching the parallel region.  An
+  // exception is when there are < WARPSIZE workers.  In this case only 1 worker
+  // is started, so we don't need a barrier.
+  if (NumThreads > 1) {
+#endif
+    named_sync(L1_BARRIER, WARPSIZE * NumWarps);
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  }
+#endif
+}
+}; // namespace
+
+EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc, int32_t gtid,
+                                                  int32_t varNum, void *array) {
+  PRINT0(LD_IO, "call to __kmpc_reduce_conditional_lastprivate(...)\n");
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+          "Expected non-SPMD mode + initialized runtime.");
+
+  omptarget_nvptx_TeamDescr &teamDescr = getMyTeamDescriptor();
+  int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  uint32_t NumThreads = GetNumberOfOmpThreads(tid, checkSPMDMode(loc),
+                                              checkRuntimeUninitialized(loc));
+  uint64_t *Buffer = teamDescr.getLastprivateIterBuffer();
+  for (unsigned i = 0; i < varNum; i++) {
+    // Reset buffer.
+    if (gtid == 0)
+      *Buffer = 0; // Reset to minimum loop iteration value.
+
+    // Barrier.
+    syncWorkersInGenericMode(NumThreads);
+
+    // Atomic max of iterations.
+    uint64_t *varArray = (uint64_t *)array;
+    uint64_t elem = varArray[i];
+    (void)atomicMax((unsigned long long int *)Buffer,
+                    (unsigned long long int)elem);
+
+    // Barrier.
+    syncWorkersInGenericMode(NumThreads);
+
+    // Read max value and update thread private array.
+    varArray[i] = *Buffer;
+
+    // Barrier.
+    syncWorkersInGenericMode(NumThreads);
+  }
+}

libomptarget/deviceRTLs/nvptx/src/omp_data.cu

@@ -0,0 +1,66 @@
+//===------------ omp_data.cu - NVPTX OpenMP GPU objects --------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the data objects used on the GPU device.
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// global device envrionment
+////////////////////////////////////////////////////////////////////////////////
+
+__device__ omptarget_device_environmentTy omptarget_device_environment;
+
+////////////////////////////////////////////////////////////////////////////////
+// global data holding OpenMP state information
+////////////////////////////////////////////////////////////////////////////////
+
+__device__
+    omptarget_nvptx_Queue<omptarget_nvptx_ThreadPrivateContext, OMP_STATE_COUNT>
+        omptarget_nvptx_device_State[MAX_SM];
+
+__device__ omptarget_nvptx_SimpleMemoryManager
+    omptarget_nvptx_simpleMemoryManager;
+__device__ __shared__ uint32_t usedMemIdx;
+__device__ __shared__ uint32_t usedSlotIdx;
+
+__device__ __shared__ uint8_t parallelLevel;
+
+// Pointer to this team's OpenMP state object
+__device__ __shared__
+    omptarget_nvptx_ThreadPrivateContext *omptarget_nvptx_threadPrivateContext;
+
+////////////////////////////////////////////////////////////////////////////////
+// The team master sets the outlined parallel function in this variable to
+// communicate with the workers.  Since it is in shared memory, there is one
+// copy of these variables for each kernel, instance, and team.
+////////////////////////////////////////////////////////////////////////////////
+volatile __device__ __shared__ omptarget_nvptx_WorkFn omptarget_nvptx_workFn;
+
+////////////////////////////////////////////////////////////////////////////////
+// OpenMP kernel execution parameters
+////////////////////////////////////////////////////////////////////////////////
+__device__ __shared__ uint32_t execution_param;
+
+////////////////////////////////////////////////////////////////////////////////
+// Data sharing state
+////////////////////////////////////////////////////////////////////////////////
+__device__ __shared__ DataSharingStateTy DataSharingState;
+
+////////////////////////////////////////////////////////////////////////////////
+// Scratchpad for teams reduction.
+////////////////////////////////////////////////////////////////////////////////
+__device__ __shared__ void *ReductionScratchpadPtr;
+
+////////////////////////////////////////////////////////////////////////////////
+// Data sharing related variables.
+////////////////////////////////////////////////////////////////////////////////
+__device__ __shared__ omptarget_nvptx_SharedArgs omptarget_nvptx_globalArgs;

libomptarget/deviceRTLs/nvptx/src/omptarget-nvptx.cu

@@ -0,0 +1,185 @@
+//===--- omptarget-nvptx.cu - NVPTX OpenMP GPU initialization ---- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the initialization code for the GPU
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// global data tables
+////////////////////////////////////////////////////////////////////////////////
+
+extern __device__
+    omptarget_nvptx_Queue<omptarget_nvptx_ThreadPrivateContext, OMP_STATE_COUNT>
+        omptarget_nvptx_device_State[MAX_SM];
+
+////////////////////////////////////////////////////////////////////////////////
+// init entry points
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE static unsigned smid() {
+  unsigned id;
+  asm("mov.u32 %0, %%smid;" : "=r"(id));
+  return id;
+}
+
+EXTERN void __kmpc_kernel_init_params(void *Ptr) {
+  PRINT(LD_IO, "call to __kmpc_kernel_init_params with version %f\n",
+        OMPTARGET_NVPTX_VERSION);
+
+  SetTeamsReductionScratchpadPtr(Ptr);
+}
+
+EXTERN void __kmpc_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime) {
+  PRINT(LD_IO, "call to __kmpc_kernel_init with version %f\n",
+        OMPTARGET_NVPTX_VERSION);
+  ASSERT0(LT_FUSSY, RequiresOMPRuntime,
+          "Generic always requires initialized runtime.");
+  setExecutionParameters(Generic, RuntimeInitialized);
+
+  int threadIdInBlock = GetThreadIdInBlock();
+  ASSERT0(LT_FUSSY, threadIdInBlock == GetMasterThreadID(),
+          "__kmpc_kernel_init() must be called by team master warp only!");
+  PRINT0(LD_IO, "call to __kmpc_kernel_init for master\n");
+
+  // Get a state object from the queue.
+  int slot = smid() % MAX_SM;
+  usedSlotIdx = slot;
+  omptarget_nvptx_threadPrivateContext =
+      omptarget_nvptx_device_State[slot].Dequeue();
+
+  // init thread private
+  int threadId = GetLogicalThreadIdInBlock(/*isSPMDExecutionMode=*/false);
+  omptarget_nvptx_threadPrivateContext->InitThreadPrivateContext(threadId);
+
+  // init team context
+  omptarget_nvptx_TeamDescr &currTeamDescr = getMyTeamDescriptor();
+  currTeamDescr.InitTeamDescr(/*isSPMDExecutionMode=*/false);
+  // this thread will start execution... has to update its task ICV
+  // to point to the level zero task ICV. That ICV was init in
+  // InitTeamDescr()
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
+      threadId, currTeamDescr.LevelZeroTaskDescr());
+
+  // set number of threads and thread limit in team to started value
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+  currTaskDescr->NThreads() = GetNumberOfWorkersInTeam();
+  currTaskDescr->ThreadLimit() = ThreadLimit;
+}
+
+EXTERN void __kmpc_kernel_deinit(int16_t IsOMPRuntimeInitialized) {
+  PRINT0(LD_IO, "call to __kmpc_kernel_deinit\n");
+  ASSERT0(LT_FUSSY, IsOMPRuntimeInitialized,
+          "Generic always requires initialized runtime.");
+  // Enqueue omp state object for use by another team.
+  int slot = usedSlotIdx;
+  omptarget_nvptx_device_State[slot].Enqueue(
+      omptarget_nvptx_threadPrivateContext);
+  // Done with work.  Kill the workers.
+  omptarget_nvptx_workFn = 0;
+}
+
+EXTERN void __kmpc_spmd_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime,
+                                    int16_t RequiresDataSharing) {
+  PRINT0(LD_IO, "call to __kmpc_spmd_kernel_init\n");
+
+  if (!RequiresOMPRuntime) {
+    // If OMP runtime is not required don't initialize OMP state.
+    setExecutionParameters(Spmd, RuntimeUninitialized);
+    if (GetThreadIdInBlock() == 0) {
+      parallelLevel = 0;
+      usedSlotIdx = smid() % MAX_SM;
+    }
+    __SYNCTHREADS();
+    return;
+  }
+  setExecutionParameters(Spmd, RuntimeInitialized);
+
+  //
+  // Team Context Initialization.
+  //
+  // In SPMD mode there is no master thread so use any cuda thread for team
+  // context initialization.
+  int threadId = GetThreadIdInBlock();
+  if (threadId == 0) {
+    // Get a state object from the queue.
+    int slot = smid() % MAX_SM;
+    usedSlotIdx = slot;
+    omptarget_nvptx_threadPrivateContext =
+        omptarget_nvptx_device_State[slot].Dequeue();
+
+    omptarget_nvptx_TeamDescr &currTeamDescr = getMyTeamDescriptor();
+    omptarget_nvptx_WorkDescr &workDescr = getMyWorkDescriptor();
+    // init team context
+    currTeamDescr.InitTeamDescr(/*isSPMDExecutionMode=*/true);
+  }
+  // FIXME: use __syncthreads instead when the function copy is fixed in LLVM.
+  __SYNCTHREADS();
+
+  omptarget_nvptx_TeamDescr &currTeamDescr = getMyTeamDescriptor();
+  omptarget_nvptx_WorkDescr &workDescr = getMyWorkDescriptor();
+
+  //
+  // Initialize task descr for each thread.
+  //
+  omptarget_nvptx_TaskDescr *newTaskDescr =
+      omptarget_nvptx_threadPrivateContext->Level1TaskDescr(threadId);
+  ASSERT0(LT_FUSSY, newTaskDescr, "expected a task descr");
+  newTaskDescr->InitLevelOneTaskDescr(ThreadLimit,
+                                      currTeamDescr.LevelZeroTaskDescr());
+  newTaskDescr->ThreadLimit() = ThreadLimit;
+  // install new top descriptor
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
+                                                             newTaskDescr);
+
+  // init thread private from init value
+  PRINT(LD_PAR,
+        "thread will execute parallel region with id %d in a team of "
+        "%d threads\n",
+        (int)newTaskDescr->ThreadId(), (int)newTaskDescr->ThreadsInTeam());
+
+  if (RequiresDataSharing && threadId % WARPSIZE == 0) {
+    // Warp master innitializes data sharing environment.
+    unsigned WID = threadId / WARPSIZE;
+    __kmpc_data_sharing_slot *RootS = currTeamDescr.RootS(
+        WID, WID == WARPSIZE - 1);
+    DataSharingState.SlotPtr[WID] = RootS;
+    DataSharingState.StackPtr[WID] = (void *)&RootS->Data[0];
+  }
+}
+
+EXTERN __attribute__((deprecated)) void __kmpc_spmd_kernel_deinit() {
+  __kmpc_spmd_kernel_deinit_v2(isRuntimeInitialized());
+}
+
+EXTERN void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime) {
+  // We're not going to pop the task descr stack of each thread since
+  // there are no more parallel regions in SPMD mode.
+  if (!RequiresOMPRuntime)
+    return;
+
+  // FIXME: use __syncthreads instead when the function copy is fixed in LLVM.
+  __SYNCTHREADS();
+  int threadId = GetThreadIdInBlock();
+  if (threadId == 0) {
+    // Enqueue omp state object for use by another team.
+    int slot = usedSlotIdx;
+    omptarget_nvptx_device_State[slot].Enqueue(
+        omptarget_nvptx_threadPrivateContext);
+  }
+}
+
+// Return true if the current target region is executed in SPMD mode.
+EXTERN int8_t __kmpc_is_spmd_exec_mode() {
+  PRINT0(LD_IO | LD_PAR, "call to __kmpc_is_spmd_exec_mode\n");
+  return isSPMDMode();
+}

libomptarget/deviceRTLs/nvptx/src/omptarget-nvptx.h

@@ -0,0 +1,443 @@
+//===---- omptarget-nvptx.h - NVPTX OpenMP GPU initialization ---- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declarations of all library macros, types,
+// and functions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __OMPTARGET_NVPTX_H
+#define __OMPTARGET_NVPTX_H
+
+// std includes
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <inttypes.h>
+
+// cuda includes
+#include <cuda.h>
+#include <math.h>
+
+// local includes
+#include "debug.h"     // debug
+#include "interface.h" // interfaces with omp, compiler, and user
+#include "option.h"    // choices we have
+#include "state-queue.h"
+#include "support.h"
+
+#define OMPTARGET_NVPTX_VERSION 1.1
+
+// used by the library for the interface with the app
+#define DISPATCH_FINISHED 0
+#define DISPATCH_NOTFINISHED 1
+
+// used by dynamic scheduling
+#define FINISHED 0
+#define NOT_FINISHED 1
+#define LAST_CHUNK 2
+
+#define BARRIER_COUNTER 0
+#define ORDERED_COUNTER 1
+
+// Macros for Cuda intrinsics
+// In Cuda 9.0, the *_sync() version takes an extra argument 'mask'.
+// Also, __ballot(1) in Cuda 8.0 is replaced with __activemask().
+#if defined(CUDART_VERSION) && CUDART_VERSION >= 9000
+#define __SHFL_SYNC(mask, var, srcLane) __shfl_sync((mask), (var), (srcLane))
+#define __SHFL_DOWN_SYNC(mask, var, delta, width)                              \
+  __shfl_down_sync((mask), (var), (delta), (width))
+#define __ACTIVEMASK() __activemask()
+#else
+#define __SHFL_SYNC(mask, var, srcLane) __shfl((var), (srcLane))
+#define __SHFL_DOWN_SYNC(mask, var, delta, width)                              \
+  __shfl_down((var), (delta), (width))
+#define __ACTIVEMASK() __ballot(1)
+#endif
+
+#define __SYNCTHREADS_N(n) asm volatile("bar.sync %0;" : : "r"(n) : "memory");
+#define __SYNCTHREADS() __SYNCTHREADS_N(0)
+
+// arguments needed for L0 parallelism only.
+class omptarget_nvptx_SharedArgs {
+public:
+  // All these methods must be called by the master thread only.
+  INLINE void Init() {
+    args  = buffer;
+    nArgs = MAX_SHARED_ARGS;
+  }
+  INLINE void DeInit() {
+    // Free any memory allocated for outlined parallel function with a large
+    // number of arguments.
+    if (nArgs > MAX_SHARED_ARGS) {
+      SafeFree(args, (char *)"new extended args");
+      Init();
+    }
+  }
+  INLINE void EnsureSize(size_t size) {
+    if (size > nArgs) {
+      if (nArgs > MAX_SHARED_ARGS) {
+        SafeFree(args, (char *)"new extended args");
+      }
+      args = (void **) SafeMalloc(size * sizeof(void *),
+                                  (char *)"new extended args");
+      nArgs = size;
+    }
+  }
+  // Called by all threads.
+  INLINE void **GetArgs() const { return args; };
+private:
+  // buffer of pre-allocated arguments.
+  void *buffer[MAX_SHARED_ARGS];
+  // pointer to arguments buffer.
+  // starts off as a pointer to 'buffer' but can be dynamically allocated.
+  void **args;
+  // starts off as MAX_SHARED_ARGS but can increase in size.
+  uint32_t nArgs;
+};
+
+extern __device__ __shared__ omptarget_nvptx_SharedArgs
+    omptarget_nvptx_globalArgs;
+
+// Data sharing related quantities, need to match what is used in the compiler.
+enum DATA_SHARING_SIZES {
+  // The maximum number of workers in a kernel.
+  DS_Max_Worker_Threads = 992,
+  // The size reserved for data in a shared memory slot.
+  DS_Slot_Size = 256,
+  // The slot size that should be reserved for a working warp.
+  DS_Worker_Warp_Slot_Size = WARPSIZE * DS_Slot_Size,
+  // The maximum number of warps in use
+  DS_Max_Warp_Number = 32,
+  // The size of the preallocated shared memory buffer per team
+  DS_Shared_Memory_Size = 128,
+};
+
+// Data structure to keep in shared memory that traces the current slot, stack,
+// and frame pointer as well as the active threads that didn't exit the current
+// environment.
+struct DataSharingStateTy {
+  __kmpc_data_sharing_slot *SlotPtr[DS_Max_Warp_Number];
+  void *StackPtr[DS_Max_Warp_Number];
+  void * volatile FramePtr[DS_Max_Warp_Number];
+  int32_t ActiveThreads[DS_Max_Warp_Number];
+};
+// Additional worker slot type which is initialized with the default worker slot
+// size of 4*32 bytes.
+struct __kmpc_data_sharing_worker_slot_static {
+  __kmpc_data_sharing_slot *Next;
+  __kmpc_data_sharing_slot *Prev;
+  void *PrevSlotStackPtr;
+  void *DataEnd;
+  char Data[DS_Worker_Warp_Slot_Size];
+};
+// Additional master slot type which is initialized with the default master slot
+// size of 4 bytes.
+struct __kmpc_data_sharing_master_slot_static {
+  __kmpc_data_sharing_slot *Next;
+  __kmpc_data_sharing_slot *Prev;
+  void *PrevSlotStackPtr;
+  void *DataEnd;
+  char Data[DS_Slot_Size];
+};
+extern __device__ __shared__ DataSharingStateTy DataSharingState;
+
+////////////////////////////////////////////////////////////////////////////////
+// task ICV and (implicit & explicit) task state
+
+class omptarget_nvptx_TaskDescr {
+public:
+  // methods for flags
+  INLINE omp_sched_t GetRuntimeSched() const;
+  INLINE void SetRuntimeSched(omp_sched_t sched);
+  INLINE int InParallelRegion() const { return items.flags & TaskDescr_InPar; }
+  INLINE int InL2OrHigherParallelRegion() const {
+    return items.flags & TaskDescr_InParL2P;
+  }
+  INLINE int IsParallelConstruct() const {
+    return items.flags & TaskDescr_IsParConstr;
+  }
+  INLINE int IsTaskConstruct() const { return !IsParallelConstruct(); }
+  // methods for other fields
+  INLINE uint16_t &NThreads() { return items.nthreads; }
+  INLINE uint16_t &ThreadLimit() { return items.threadlimit; }
+  INLINE uint16_t &ThreadId() { return items.threadId; }
+  INLINE uint16_t &ThreadsInTeam() { return items.threadsInTeam; }
+  INLINE uint64_t &RuntimeChunkSize() { return items.runtimeChunkSize; }
+  INLINE omptarget_nvptx_TaskDescr *GetPrevTaskDescr() const { return prev; }
+  INLINE void SetPrevTaskDescr(omptarget_nvptx_TaskDescr *taskDescr) {
+    prev = taskDescr;
+  }
+  // init & copy
+  INLINE void InitLevelZeroTaskDescr(bool isSPMDExecutionMode);
+  INLINE void InitLevelOneTaskDescr(uint16_t tnum,
+                                    omptarget_nvptx_TaskDescr *parentTaskDescr);
+  INLINE void Copy(omptarget_nvptx_TaskDescr *sourceTaskDescr);
+  INLINE void CopyData(omptarget_nvptx_TaskDescr *sourceTaskDescr);
+  INLINE void CopyParent(omptarget_nvptx_TaskDescr *parentTaskDescr);
+  INLINE void CopyForExplicitTask(omptarget_nvptx_TaskDescr *parentTaskDescr);
+  INLINE void CopyToWorkDescr(omptarget_nvptx_TaskDescr *masterTaskDescr,
+                              uint16_t tnum);
+  INLINE void CopyFromWorkDescr(omptarget_nvptx_TaskDescr *workTaskDescr);
+  INLINE void CopyConvergentParent(omptarget_nvptx_TaskDescr *parentTaskDescr,
+                                   uint16_t tid, uint16_t tnum);
+  INLINE void SaveLoopData();
+  INLINE void RestoreLoopData() const;
+
+private:
+  // bits for flags: (6 used, 2 free)
+  //   3 bits (SchedMask) for runtime schedule
+  //   1 bit (InPar) if this thread has encountered one or more parallel region
+  //   1 bit (IsParConstr) if ICV for a parallel region (false = explicit task)
+  //   1 bit (InParL2+) if this thread has encountered L2 or higher parallel
+  //   region
+  static const uint8_t TaskDescr_SchedMask = (0x1 | 0x2 | 0x4);
+  static const uint8_t TaskDescr_InPar = 0x10;
+  static const uint8_t TaskDescr_IsParConstr = 0x20;
+  static const uint8_t TaskDescr_InParL2P = 0x40;
+
+  struct SavedLoopDescr_items {
+    int64_t loopUpperBound;
+    int64_t nextLowerBound;
+    int64_t chunk;
+    int64_t stride;
+    kmp_sched_t schedule;
+  } loopData;
+
+  struct TaskDescr_items {
+    uint8_t flags; // 6 bit used (see flag above)
+    uint8_t unused;
+    uint16_t nthreads;         // thread num for subsequent parallel regions
+    uint16_t threadlimit;      // thread limit ICV
+    uint16_t threadId;         // thread id
+    uint16_t threadsInTeam;    // threads in current team
+    uint64_t runtimeChunkSize; // runtime chunk size
+  } items;
+  omptarget_nvptx_TaskDescr *prev;
+};
+
+// build on kmp
+typedef struct omptarget_nvptx_ExplicitTaskDescr {
+  omptarget_nvptx_TaskDescr
+      taskDescr; // omptarget_nvptx task description (must be first)
+  kmp_TaskDescr kmpTaskDescr; // kmp task description (must be last)
+} omptarget_nvptx_ExplicitTaskDescr;
+
+////////////////////////////////////////////////////////////////////////////////
+// Descriptor of a parallel region (worksharing in general)
+
+class omptarget_nvptx_WorkDescr {
+
+public:
+  // access to data
+  INLINE omptarget_nvptx_TaskDescr *WorkTaskDescr() { return &masterTaskICV; }
+
+private:
+  omptarget_nvptx_TaskDescr masterTaskICV;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class omptarget_nvptx_TeamDescr {
+public:
+  // access to data
+  INLINE omptarget_nvptx_TaskDescr *LevelZeroTaskDescr() {
+    return &levelZeroTaskDescr;
+  }
+  INLINE omptarget_nvptx_WorkDescr &WorkDescr() {
+    return workDescrForActiveParallel;
+  }
+  INLINE uint64_t *getLastprivateIterBuffer() { return &lastprivateIterBuffer; }
+
+  // init
+  INLINE void InitTeamDescr(bool isSPMDExecutionMode);
+
+  INLINE __kmpc_data_sharing_slot *RootS(int wid, bool IsMasterThread) {
+    // If this is invoked by the master thread of the master warp then intialize
+    // it with a smaller slot.
+    if (IsMasterThread) {
+      // Do not initalize this slot again if it has already been initalized.
+      if (master_rootS[0].DataEnd == &master_rootS[0].Data[0] + DS_Slot_Size)
+        return 0;
+      // Initialize the pointer to the end of the slot given the size of the
+      // data section. DataEnd is non-inclusive.
+      master_rootS[0].DataEnd = &master_rootS[0].Data[0] + DS_Slot_Size;
+      // We currently do not have a next slot.
+      master_rootS[0].Next = 0;
+      master_rootS[0].Prev = 0;
+      master_rootS[0].PrevSlotStackPtr = 0;
+      return (__kmpc_data_sharing_slot *)&master_rootS[0];
+    }
+    // Do not initalize this slot again if it has already been initalized.
+    if (worker_rootS[wid].DataEnd ==
+        &worker_rootS[wid].Data[0] + DS_Worker_Warp_Slot_Size)
+      return 0;
+    // Initialize the pointer to the end of the slot given the size of the data
+    // section. DataEnd is non-inclusive.
+    worker_rootS[wid].DataEnd =
+        &worker_rootS[wid].Data[0] + DS_Worker_Warp_Slot_Size;
+    // We currently do not have a next slot.
+    worker_rootS[wid].Next = 0;
+    worker_rootS[wid].Prev = 0;
+    worker_rootS[wid].PrevSlotStackPtr = 0;
+    return (__kmpc_data_sharing_slot *)&worker_rootS[wid];
+  }
+
+  INLINE __kmpc_data_sharing_slot *GetPreallocatedSlotAddr(int wid) {
+    worker_rootS[wid].DataEnd =
+        &worker_rootS[wid].Data[0] + DS_Worker_Warp_Slot_Size;
+    // We currently do not have a next slot.
+    worker_rootS[wid].Next = 0;
+    worker_rootS[wid].Prev = 0;
+    worker_rootS[wid].PrevSlotStackPtr = 0;
+    return (__kmpc_data_sharing_slot *)&worker_rootS[wid];
+  }
+
+private:
+  omptarget_nvptx_TaskDescr
+      levelZeroTaskDescr; // icv for team master initial thread
+  omptarget_nvptx_WorkDescr
+      workDescrForActiveParallel; // one, ONLY for the active par
+  uint64_t lastprivateIterBuffer;
+
+  __align__(16)
+      __kmpc_data_sharing_worker_slot_static worker_rootS[WARPSIZE];
+  __align__(16) __kmpc_data_sharing_master_slot_static master_rootS[1];
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// thread private data (struct of arrays for better coalescing)
+// tid refers here to the global thread id
+// do not support multiple concurrent kernel a this time
+class omptarget_nvptx_ThreadPrivateContext {
+public:
+  // task
+  INLINE omptarget_nvptx_TaskDescr *Level1TaskDescr(int tid) {
+    return &levelOneTaskDescr[tid];
+  }
+  INLINE void SetTopLevelTaskDescr(int tid,
+                                   omptarget_nvptx_TaskDescr *taskICV) {
+    topTaskDescr[tid] = taskICV;
+  }
+  INLINE omptarget_nvptx_TaskDescr *GetTopLevelTaskDescr(int tid) const;
+  // parallel
+  INLINE uint16_t &NumThreadsForNextParallel(int tid) {
+    return nextRegion.tnum[tid];
+  }
+  // simd
+  INLINE uint16_t &SimdLimitForNextSimd(int tid) {
+    return nextRegion.slim[tid];
+  }
+  // schedule (for dispatch)
+  INLINE kmp_sched_t &ScheduleType(int tid) { return schedule[tid]; }
+  INLINE int64_t &Chunk(int tid) { return chunk[tid]; }
+  INLINE int64_t &LoopUpperBound(int tid) { return loopUpperBound[tid]; }
+  INLINE int64_t &NextLowerBound(int tid) { return nextLowerBound[tid]; }
+  INLINE int64_t &Stride(int tid) { return stride[tid]; }
+
+  INLINE omptarget_nvptx_TeamDescr &TeamContext() { return teamContext; }
+
+  INLINE void InitThreadPrivateContext(int tid);
+  INLINE uint64_t &Cnt() { return cnt; }
+
+private:
+  // team context for this team
+  omptarget_nvptx_TeamDescr teamContext;
+  // task ICV for implict threads in the only parallel region
+  omptarget_nvptx_TaskDescr levelOneTaskDescr[MAX_THREADS_PER_TEAM];
+  // pointer where to find the current task ICV (top of the stack)
+  omptarget_nvptx_TaskDescr *topTaskDescr[MAX_THREADS_PER_TEAM];
+  union {
+    // Only one of the two is live at the same time.
+    // parallel
+    uint16_t tnum[MAX_THREADS_PER_TEAM];
+    // simd limit
+    uint16_t slim[MAX_THREADS_PER_TEAM];
+  } nextRegion;
+  // schedule (for dispatch)
+  kmp_sched_t schedule[MAX_THREADS_PER_TEAM]; // remember schedule type for #for
+  int64_t chunk[MAX_THREADS_PER_TEAM];
+  int64_t loopUpperBound[MAX_THREADS_PER_TEAM];
+  // state for dispatch with dyn/guided OR static (never use both at a time)
+  int64_t nextLowerBound[MAX_THREADS_PER_TEAM];
+  int64_t stride[MAX_THREADS_PER_TEAM];
+  uint64_t cnt;
+};
+
+/// Device envrionment data
+struct omptarget_device_environmentTy {
+  int32_t debug_level;
+};
+
+/// Memory manager for statically allocated memory.
+class omptarget_nvptx_SimpleMemoryManager {
+private:
+  __align__(128) struct MemDataTy {
+    volatile unsigned keys[OMP_STATE_COUNT];
+  } MemData[MAX_SM];
+
+  INLINE static uint32_t hash(unsigned key) {
+    return key & (OMP_STATE_COUNT - 1);
+  }
+
+public:
+  INLINE void Release();
+  INLINE const void *Acquire(const void *buf, size_t size);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// global device envrionment
+////////////////////////////////////////////////////////////////////////////////
+
+extern __device__ omptarget_device_environmentTy omptarget_device_environment;
+
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// global data tables
+////////////////////////////////////////////////////////////////////////////////
+
+extern __device__ omptarget_nvptx_SimpleMemoryManager
+    omptarget_nvptx_simpleMemoryManager;
+extern __device__ __shared__ uint32_t usedMemIdx;
+extern __device__ __shared__ uint32_t usedSlotIdx;
+extern __device__ __shared__ uint8_t parallelLevel;
+extern __device__ __shared__
+    omptarget_nvptx_ThreadPrivateContext *omptarget_nvptx_threadPrivateContext;
+
+extern __device__ __shared__ uint32_t execution_param;
+extern __device__ __shared__ void *ReductionScratchpadPtr;
+
+////////////////////////////////////////////////////////////////////////////////
+// work function (outlined parallel/simd functions) and arguments.
+// needed for L1 parallelism only.
+////////////////////////////////////////////////////////////////////////////////
+
+typedef void *omptarget_nvptx_WorkFn;
+extern volatile __device__ __shared__ omptarget_nvptx_WorkFn
+    omptarget_nvptx_workFn;
+
+////////////////////////////////////////////////////////////////////////////////
+// get private data structures
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE omptarget_nvptx_TeamDescr &getMyTeamDescriptor();
+INLINE omptarget_nvptx_WorkDescr &getMyWorkDescriptor();
+INLINE omptarget_nvptx_TaskDescr *
+getMyTopTaskDescriptor(bool isSPMDExecutionMode);
+INLINE omptarget_nvptx_TaskDescr *getMyTopTaskDescriptor(int globalThreadId);
+
+////////////////////////////////////////////////////////////////////////////////
+// inlined implementation
+////////////////////////////////////////////////////////////////////////////////
+
+#include "omptarget-nvptxi.h"
+#include "supporti.h"
+
+#endif

libomptarget/deviceRTLs/nvptx/src/omptarget-nvptxi.h

@@ -0,0 +1,234 @@
+//===---- omptarget-nvptxi.h - NVPTX OpenMP GPU initialization --- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declarations of all library macros, types,
+// and functions.
+//
+//===----------------------------------------------------------------------===//
+
+////////////////////////////////////////////////////////////////////////////////
+// Task Descriptor
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE omp_sched_t omptarget_nvptx_TaskDescr::GetRuntimeSched() const {
+  // sched starts from 1..4; encode it as 0..3; so add 1 here
+  uint8_t rc = (items.flags & TaskDescr_SchedMask) + 1;
+  return (omp_sched_t)rc;
+}
+
+INLINE void omptarget_nvptx_TaskDescr::SetRuntimeSched(omp_sched_t sched) {
+  // sched starts from 1..4; encode it as 0..3; so sub 1 here
+  uint8_t val = ((uint8_t)sched) - 1;
+  // clear current sched
+  items.flags &= ~TaskDescr_SchedMask;
+  // set new sched
+  items.flags |= val;
+}
+
+INLINE void
+omptarget_nvptx_TaskDescr::InitLevelZeroTaskDescr(bool isSPMDExecutionMode) {
+  // slow method
+  // flag:
+  //   default sched is static,
+  //   dyn is off (unused now anyway, but may need to sample from host ?)
+  //   not in parallel
+
+  items.flags = 0;
+  items.nthreads = GetNumberOfProcsInTeam(isSPMDExecutionMode);
+  ;                                // threads: whatever was alloc by kernel
+  items.threadId = 0;         // is master
+  items.threadsInTeam = 1;    // sequential
+  items.runtimeChunkSize = 1; // prefered chunking statik with chunk 1
+}
+
+// This is called when all threads are started together in SPMD mode.
+// OMP directives include target parallel, target distribute parallel for, etc.
+INLINE void omptarget_nvptx_TaskDescr::InitLevelOneTaskDescr(
+    uint16_t tnum, omptarget_nvptx_TaskDescr *parentTaskDescr) {
+  // slow method
+  // flag:
+  //   default sched is static,
+  //   dyn is off (unused now anyway, but may need to sample from host ?)
+  //   in L1 parallel
+
+  items.flags =
+      TaskDescr_InPar | TaskDescr_IsParConstr; // set flag to parallel
+  items.nthreads = 0; // # threads for subsequent parallel region
+  items.threadId =
+      GetThreadIdInBlock(); // get ids from cuda (only called for 1st level)
+  items.threadsInTeam = tnum;
+  items.runtimeChunkSize = 1; // prefered chunking statik with chunk 1
+  prev = parentTaskDescr;
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyData(
+    omptarget_nvptx_TaskDescr *sourceTaskDescr) {
+  items = sourceTaskDescr->items;
+}
+
+INLINE void
+omptarget_nvptx_TaskDescr::Copy(omptarget_nvptx_TaskDescr *sourceTaskDescr) {
+  CopyData(sourceTaskDescr);
+  prev = sourceTaskDescr->prev;
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyParent(
+    omptarget_nvptx_TaskDescr *parentTaskDescr) {
+  CopyData(parentTaskDescr);
+  prev = parentTaskDescr;
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyForExplicitTask(
+    omptarget_nvptx_TaskDescr *parentTaskDescr) {
+  CopyParent(parentTaskDescr);
+  items.flags = items.flags & ~TaskDescr_IsParConstr;
+  ASSERT0(LT_FUSSY, IsTaskConstruct(), "expected task");
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyToWorkDescr(
+    omptarget_nvptx_TaskDescr *masterTaskDescr, uint16_t tnum) {
+  CopyParent(masterTaskDescr);
+  // overrwrite specific items;
+  items.flags |=
+      TaskDescr_InPar | TaskDescr_IsParConstr; // set flag to parallel
+  items.threadsInTeam = tnum;             // set number of threads
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyFromWorkDescr(
+    omptarget_nvptx_TaskDescr *workTaskDescr) {
+  Copy(workTaskDescr);
+  //
+  // overrwrite specific items;
+  //
+  // The threadID should be GetThreadIdInBlock() % GetMasterThreadID().
+  // This is so that the serial master (first lane in the master warp)
+  // gets a threadId of 0.
+  // However, we know that this function is always called in a parallel
+  // region where only workers are active.  The serial master thread
+  // never enters this region.  When a parallel region is executed serially,
+  // the threadId is set to 0 elsewhere and the kmpc_serialized_* functions
+  // are called, which never activate this region.
+  items.threadId =
+      GetThreadIdInBlock(); // get ids from cuda (only called for 1st level)
+}
+
+INLINE void omptarget_nvptx_TaskDescr::CopyConvergentParent(
+    omptarget_nvptx_TaskDescr *parentTaskDescr, uint16_t tid, uint16_t tnum) {
+  CopyParent(parentTaskDescr);
+  items.flags |= TaskDescr_InParL2P; // In L2+ parallelism
+  items.threadsInTeam = tnum;        // set number of threads
+  items.threadId = tid;
+}
+
+INLINE void omptarget_nvptx_TaskDescr::SaveLoopData() {
+  loopData.loopUpperBound =
+      omptarget_nvptx_threadPrivateContext->LoopUpperBound(items.threadId);
+  loopData.nextLowerBound =
+      omptarget_nvptx_threadPrivateContext->NextLowerBound(items.threadId);
+  loopData.schedule =
+      omptarget_nvptx_threadPrivateContext->ScheduleType(items.threadId);
+  loopData.chunk = omptarget_nvptx_threadPrivateContext->Chunk(items.threadId);
+  loopData.stride =
+      omptarget_nvptx_threadPrivateContext->Stride(items.threadId);
+}
+
+INLINE void omptarget_nvptx_TaskDescr::RestoreLoopData() const {
+  omptarget_nvptx_threadPrivateContext->Chunk(items.threadId) = loopData.chunk;
+  omptarget_nvptx_threadPrivateContext->LoopUpperBound(items.threadId) =
+      loopData.loopUpperBound;
+  omptarget_nvptx_threadPrivateContext->NextLowerBound(items.threadId) =
+      loopData.nextLowerBound;
+  omptarget_nvptx_threadPrivateContext->Stride(items.threadId) =
+      loopData.stride;
+  omptarget_nvptx_threadPrivateContext->ScheduleType(items.threadId) =
+      loopData.schedule;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Thread Private Context
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE omptarget_nvptx_TaskDescr *
+omptarget_nvptx_ThreadPrivateContext::GetTopLevelTaskDescr(int tid) const {
+  ASSERT0(
+      LT_FUSSY, tid < MAX_THREADS_PER_TEAM,
+      "Getting top level, tid is larger than allocated data structure size");
+  return topTaskDescr[tid];
+}
+
+INLINE void
+omptarget_nvptx_ThreadPrivateContext::InitThreadPrivateContext(int tid) {
+  // levelOneTaskDescr is init when starting the parallel region
+  // top task descr is NULL (team master version will be fixed separately)
+  topTaskDescr[tid] = NULL;
+  // no num threads value has been pushed
+  nextRegion.tnum[tid] = 0;
+  // the following don't need to be init here; they are init when using dyn
+  // sched
+  // current_Event, events_Number, chunk, num_Iterations, schedule
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Team Descriptor
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE void omptarget_nvptx_TeamDescr::InitTeamDescr(bool isSPMDExecutionMode) {
+  levelZeroTaskDescr.InitLevelZeroTaskDescr(isSPMDExecutionMode);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Get private data structure for thread
+////////////////////////////////////////////////////////////////////////////////
+
+// Utility routines for CUDA threads
+INLINE omptarget_nvptx_TeamDescr &getMyTeamDescriptor() {
+  return omptarget_nvptx_threadPrivateContext->TeamContext();
+}
+
+INLINE omptarget_nvptx_WorkDescr &getMyWorkDescriptor() {
+  omptarget_nvptx_TeamDescr &currTeamDescr = getMyTeamDescriptor();
+  return currTeamDescr.WorkDescr();
+}
+
+INLINE omptarget_nvptx_TaskDescr *getMyTopTaskDescriptor(int threadId) {
+  return omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+}
+
+INLINE omptarget_nvptx_TaskDescr *
+getMyTopTaskDescriptor(bool isSPMDExecutionMode) {
+  return getMyTopTaskDescriptor(GetLogicalThreadIdInBlock(isSPMDExecutionMode));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Memory management runtime functions.
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE void omptarget_nvptx_SimpleMemoryManager::Release() {
+  ASSERT0(LT_FUSSY, usedSlotIdx < MAX_SM,
+          "SlotIdx is too big or uninitialized.");
+  ASSERT0(LT_FUSSY, usedMemIdx < OMP_STATE_COUNT,
+          "MemIdx is too big or uninitialized.");
+  MemDataTy &MD = MemData[usedSlotIdx];
+  atomicExch((unsigned *)&MD.keys[usedMemIdx], 0);
+}
+
+INLINE const void *omptarget_nvptx_SimpleMemoryManager::Acquire(const void *buf,
+                                                                size_t size) {
+  ASSERT0(LT_FUSSY, usedSlotIdx < MAX_SM,
+          "SlotIdx is too big or uninitialized.");
+  const unsigned sm = usedSlotIdx;
+  MemDataTy &MD = MemData[sm];
+  unsigned i = hash(GetBlockIdInKernel());
+  while (atomicCAS((unsigned *)&MD.keys[i], 0, 1) != 0) {
+    i = hash(i + 1);
+  }
+  usedSlotIdx = sm;
+  usedMemIdx = i;
+  return static_cast<const char *>(buf) + (sm * OMP_STATE_COUNT + i) * size;
+}

libomptarget/deviceRTLs/nvptx/src/option.h

@@ -0,0 +1,66 @@
+//===------------ option.h - NVPTX OpenMP GPU options ------------ CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// GPU default options
+//
+//===----------------------------------------------------------------------===//
+#ifndef _OPTION_H_
+#define _OPTION_H_
+
+////////////////////////////////////////////////////////////////////////////////
+// Kernel options
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// The following def must match the absolute limit hardwired in the host RTL
+// max number of threads per team
+#define MAX_THREADS_PER_TEAM 1024
+
+#define WARPSIZE 32
+
+// The named barrier for active parallel threads of a team in an L1 parallel
+// region to synchronize with each other.
+#define L1_BARRIER (1)
+
+// Maximum number of preallocated arguments to an outlined parallel/simd function.
+// Anything more requires dynamic memory allocation.
+#define MAX_SHARED_ARGS 20
+
+// Maximum number of omp state objects per SM allocated statically in global
+// memory.
+#if __CUDA_ARCH__ >= 700
+#define OMP_STATE_COUNT 32
+#define MAX_SM 84
+#elif __CUDA_ARCH__ >= 600
+#define OMP_STATE_COUNT 32
+#define MAX_SM 56
+#else
+#define OMP_STATE_COUNT 16
+#define MAX_SM 16
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// algo options
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+// misc options (by def everythig here is device)
+////////////////////////////////////////////////////////////////////////////////
+
+#define EXTERN extern "C" __device__
+#define INLINE __inline__ __device__
+#define NOINLINE __noinline__ __device__
+#ifndef TRUE
+#define TRUE 1
+#endif
+#ifndef FALSE
+#define FALSE 0
+#endif
+
+#endif

libomptarget/deviceRTLs/nvptx/src/parallel.cu

@@ -0,0 +1,469 @@
+//===---- parallel.cu - NVPTX OpenMP parallel implementation ----- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Parallel implemention in the GPU. Here is the pattern:
+//
+//    while (not finished) {
+//
+//    if (master) {
+//      sequential code, decide which par loop to do, or if finished
+//     __kmpc_kernel_prepare_parallel() // exec by master only
+//    }
+//    syncthreads // A
+//    __kmpc_kernel_parallel() // exec by all
+//    if (this thread is included in the parallel) {
+//      switch () for all parallel loops
+//      __kmpc_kernel_end_parallel() // exec only by threads in parallel
+//    }
+//
+//
+//    The reason we don't exec end_parallel for the threads not included
+//    in the parallel loop is that for each barrier in the parallel
+//    region, these non-included threads will cycle through the
+//    syncthread A. Thus they must preserve their current threadId that
+//    is larger than thread in team.
+//
+//    To make a long story short...
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+typedef struct ConvergentSimdJob {
+  omptarget_nvptx_TaskDescr taskDescr;
+  omptarget_nvptx_TaskDescr *convHeadTaskDescr;
+  uint16_t slimForNextSimd;
+} ConvergentSimdJob;
+
+////////////////////////////////////////////////////////////////////////////////
+// support for convergent simd (team of threads in a warp only)
+////////////////////////////////////////////////////////////////////////////////
+EXTERN bool __kmpc_kernel_convergent_simd(void *buffer, uint32_t Mask,
+                                          bool *IsFinal, int32_t *LaneSource,
+                                          int32_t *LaneId, int32_t *NumLanes) {
+  PRINT0(LD_IO, "call to __kmpc_kernel_convergent_simd\n");
+  uint32_t ConvergentMask = Mask;
+  int32_t ConvergentSize = __popc(ConvergentMask);
+  uint32_t WorkRemaining = ConvergentMask >> (*LaneSource + 1);
+  *LaneSource += __ffs(WorkRemaining);
+  *IsFinal = __popc(WorkRemaining) == 1;
+  uint32_t lanemask_lt;
+  asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lanemask_lt));
+  *LaneId = __popc(ConvergentMask & lanemask_lt);
+
+  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
+  int sourceThreadId = (threadId & ~(WARPSIZE - 1)) + *LaneSource;
+
+  ConvergentSimdJob *job = (ConvergentSimdJob *)buffer;
+  int32_t SimdLimit =
+      omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId);
+  job->slimForNextSimd = SimdLimit;
+
+  int32_t SimdLimitSource = __SHFL_SYNC(Mask, SimdLimit, *LaneSource);
+  // reset simdlimit to avoid propagating to successive #simd
+  if (SimdLimitSource > 0 && threadId == sourceThreadId)
+    omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId) = 0;
+
+  // We cannot have more than the # of convergent threads.
+  if (SimdLimitSource > 0)
+    *NumLanes = min(ConvergentSize, SimdLimitSource);
+  else
+    *NumLanes = ConvergentSize;
+  ASSERT(LT_FUSSY, *NumLanes > 0, "bad thread request of %d threads",
+         (int)*NumLanes);
+
+  // Set to true for lanes participating in the simd region.
+  bool isActive = false;
+  // Initialize state for active threads.
+  if (*LaneId < *NumLanes) {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+    omptarget_nvptx_TaskDescr *sourceTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(
+            sourceThreadId);
+    job->convHeadTaskDescr = currTaskDescr;
+    // install top descriptor from the thread for which the lanes are working.
+    omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
+                                                               sourceTaskDescr);
+    isActive = true;
+  }
+
+  // requires a memory fence between threads of a warp
+  return isActive;
+}
+
+EXTERN void __kmpc_kernel_end_convergent_simd(void *buffer) {
+  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_end_convergent_parallel\n");
+  // pop stack
+  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
+  ConvergentSimdJob *job = (ConvergentSimdJob *)buffer;
+  omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(threadId) =
+      job->slimForNextSimd;
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
+      threadId, job->convHeadTaskDescr);
+}
+
+typedef struct ConvergentParallelJob {
+  omptarget_nvptx_TaskDescr taskDescr;
+  omptarget_nvptx_TaskDescr *convHeadTaskDescr;
+  uint16_t tnumForNextPar;
+} ConvergentParallelJob;
+
+////////////////////////////////////////////////////////////////////////////////
+// support for convergent parallelism (team of threads in a warp only)
+////////////////////////////////////////////////////////////////////////////////
+EXTERN bool __kmpc_kernel_convergent_parallel(void *buffer, uint32_t Mask,
+                                              bool *IsFinal,
+                                              int32_t *LaneSource) {
+  PRINT0(LD_IO, "call to __kmpc_kernel_convergent_parallel\n");
+  uint32_t ConvergentMask = Mask;
+  int32_t ConvergentSize = __popc(ConvergentMask);
+  uint32_t WorkRemaining = ConvergentMask >> (*LaneSource + 1);
+  *LaneSource += __ffs(WorkRemaining);
+  *IsFinal = __popc(WorkRemaining) == 1;
+  uint32_t lanemask_lt;
+  asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lanemask_lt));
+  uint32_t OmpId = __popc(ConvergentMask & lanemask_lt);
+
+  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
+  int sourceThreadId = (threadId & ~(WARPSIZE - 1)) + *LaneSource;
+
+  ConvergentParallelJob *job = (ConvergentParallelJob *)buffer;
+  int32_t NumThreadsClause =
+      omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId);
+  job->tnumForNextPar = NumThreadsClause;
+
+  int32_t NumThreadsSource = __SHFL_SYNC(Mask, NumThreadsClause, *LaneSource);
+  // reset numthreads to avoid propagating to successive #parallel
+  if (NumThreadsSource > 0 && threadId == sourceThreadId)
+    omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId) =
+        0;
+
+  // We cannot have more than the # of convergent threads.
+  uint16_t NumThreads;
+  if (NumThreadsSource > 0)
+    NumThreads = min(ConvergentSize, NumThreadsSource);
+  else
+    NumThreads = ConvergentSize;
+  ASSERT(LT_FUSSY, NumThreads > 0, "bad thread request of %d threads",
+         (int)NumThreads);
+
+  // Set to true for workers participating in the parallel region.
+  bool isActive = false;
+  // Initialize state for active threads.
+  if (OmpId < NumThreads) {
+    // init L2 task descriptor and storage for the L1 parallel task descriptor.
+    omptarget_nvptx_TaskDescr *newTaskDescr = &job->taskDescr;
+    ASSERT0(LT_FUSSY, newTaskDescr, "expected a task descr");
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+    omptarget_nvptx_TaskDescr *sourceTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(
+            sourceThreadId);
+    job->convHeadTaskDescr = currTaskDescr;
+    newTaskDescr->CopyConvergentParent(sourceTaskDescr, OmpId, NumThreads);
+    // install new top descriptor
+    omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
+                                                               newTaskDescr);
+    isActive = true;
+  }
+
+  // requires a memory fence between threads of a warp
+  return isActive;
+}
+
+EXTERN void __kmpc_kernel_end_convergent_parallel(void *buffer) {
+  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_end_convergent_parallel\n");
+  // pop stack
+  int threadId = GetLogicalThreadIdInBlock(isSPMDMode());
+  ConvergentParallelJob *job = (ConvergentParallelJob *)buffer;
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
+      threadId, job->convHeadTaskDescr);
+  omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId) =
+      job->tnumForNextPar;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// support for parallel that goes parallel (1 static level only)
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE static uint16_t determineNumberOfThreads(uint16_t NumThreadsClause,
+                                                uint16_t NThreadsICV,
+                                                uint16_t ThreadLimit) {
+  uint16_t ThreadsRequested = NThreadsICV;
+  if (NumThreadsClause != 0) {
+    ThreadsRequested = NumThreadsClause;
+  }
+
+  uint16_t ThreadsAvailable = GetNumberOfWorkersInTeam();
+  if (ThreadLimit != 0 && ThreadLimit < ThreadsAvailable) {
+    ThreadsAvailable = ThreadLimit;
+  }
+
+  uint16_t NumThreads = ThreadsAvailable;
+  if (ThreadsRequested != 0 && ThreadsRequested < NumThreads) {
+    NumThreads = ThreadsRequested;
+  }
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  // On Volta and newer architectures we require that all lanes in
+  // a warp participate in the parallel region.  Round down to a
+  // multiple of WARPSIZE since it is legal to do so in OpenMP.
+  if (NumThreads < WARPSIZE) {
+    NumThreads = 1;
+  } else {
+    NumThreads = (NumThreads & ~((uint16_t)WARPSIZE - 1));
+  }
+#endif
+
+  return NumThreads;
+}
+
+// This routine is always called by the team master..
+EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn,
+                                           int16_t IsOMPRuntimeInitialized) {
+  PRINT0(LD_IO, "call to __kmpc_kernel_prepare_parallel\n");
+  ASSERT0(LT_FUSSY, IsOMPRuntimeInitialized, "Expected initialized runtime.");
+
+  omptarget_nvptx_workFn = WorkFn;
+
+  // This routine is only called by the team master.  The team master is
+  // the first thread of the last warp.  It always has the logical thread
+  // id of 0 (since it is a shadow for the first worker thread).
+  const int threadId = 0;
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+  ASSERT0(LT_FUSSY, currTaskDescr, "expected a top task descr");
+  ASSERT0(LT_FUSSY, !currTaskDescr->InParallelRegion(),
+          "cannot be called in a parallel region.");
+  if (currTaskDescr->InParallelRegion()) {
+    PRINT0(LD_PAR, "already in parallel: go seq\n");
+    return;
+  }
+
+  uint16_t &NumThreadsClause =
+      omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(threadId);
+
+  uint16_t NumThreads =
+      determineNumberOfThreads(NumThreadsClause, currTaskDescr->NThreads(),
+                               currTaskDescr->ThreadLimit());
+
+  if (NumThreadsClause != 0) {
+    // Reset request to avoid propagating to successive #parallel
+    NumThreadsClause = 0;
+  }
+
+  ASSERT(LT_FUSSY, NumThreads > 0, "bad thread request of %d threads",
+         (int)NumThreads);
+  ASSERT0(LT_FUSSY, GetThreadIdInBlock() == GetMasterThreadID(),
+          "only team master can create parallel");
+
+  // Set number of threads on work descriptor.
+  omptarget_nvptx_WorkDescr &workDescr = getMyWorkDescriptor();
+  workDescr.WorkTaskDescr()->CopyToWorkDescr(currTaskDescr, NumThreads);
+}
+
+// All workers call this function.  Deactivate those not needed.
+// Fn - the outlined work function to execute.
+// returns True if this thread is active, else False.
+//
+// Only the worker threads call this routine.
+EXTERN bool __kmpc_kernel_parallel(void **WorkFn,
+                                   int16_t IsOMPRuntimeInitialized) {
+  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_parallel\n");
+
+  ASSERT0(LT_FUSSY, IsOMPRuntimeInitialized, "Expected initialized runtime.");
+
+  // Work function and arguments for L1 parallel region.
+  *WorkFn = omptarget_nvptx_workFn;
+
+  // If this is the termination signal from the master, quit early.
+  if (!*WorkFn) {
+    PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_parallel finished\n");
+    return false;
+  }
+
+  // Only the worker threads call this routine and the master warp
+  // never arrives here.  Therefore, use the nvptx thread id.
+  int threadId = GetThreadIdInBlock();
+  omptarget_nvptx_WorkDescr &workDescr = getMyWorkDescriptor();
+  // Set to true for workers participating in the parallel region.
+  bool isActive = false;
+  // Initialize state for active threads.
+  if (threadId < workDescr.WorkTaskDescr()->ThreadsInTeam()) {
+    // init work descriptor from workdesccr
+    omptarget_nvptx_TaskDescr *newTaskDescr =
+        omptarget_nvptx_threadPrivateContext->Level1TaskDescr(threadId);
+    ASSERT0(LT_FUSSY, newTaskDescr, "expected a task descr");
+    newTaskDescr->CopyFromWorkDescr(workDescr.WorkTaskDescr());
+    // install new top descriptor
+    omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
+                                                               newTaskDescr);
+    // init private from int value
+    PRINT(LD_PAR,
+          "thread will execute parallel region with id %d in a team of "
+          "%d threads\n",
+          (int)newTaskDescr->ThreadId(), (int)newTaskDescr->NThreads());
+
+    isActive = true;
+  }
+
+  return isActive;
+}
+
+EXTERN void __kmpc_kernel_end_parallel() {
+  // pop stack
+  PRINT0(LD_IO | LD_PAR, "call to __kmpc_kernel_end_parallel\n");
+  ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
+
+  // Only the worker threads call this routine and the master warp
+  // never arrives here.  Therefore, use the nvptx thread id.
+  int threadId = GetThreadIdInBlock();
+  omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(threadId);
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
+      threadId, currTaskDescr->GetPrevTaskDescr());
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// support for parallel that goes sequential
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN void __kmpc_serialized_parallel(kmp_Ident *loc, uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_serialized_parallel\n");
+
+  if (checkRuntimeUninitialized(loc)) {
+    ASSERT0(LT_FUSSY, checkSPMDMode(loc),
+            "Expected SPMD mode with uninitialized runtime.");
+    __SYNCTHREADS();
+    if (GetThreadIdInBlock() == 0)
+      ++parallelLevel;
+    __SYNCTHREADS();
+
+    return;
+  }
+
+  // assume this is only called for nested parallel
+  int threadId = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+
+  // unlike actual parallel, threads in the same team do not share
+  // the workTaskDescr in this case and num threads is fixed to 1
+
+  // get current task
+  omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(threadId);
+  currTaskDescr->SaveLoopData();
+
+  // allocate new task descriptor and copy value from current one, set prev to
+  // it
+  omptarget_nvptx_TaskDescr *newTaskDescr =
+      (omptarget_nvptx_TaskDescr *)SafeMalloc(sizeof(omptarget_nvptx_TaskDescr),
+                                              "new seq parallel task");
+  newTaskDescr->CopyParent(currTaskDescr);
+
+  // tweak values for serialized parallel case:
+  // - each thread becomes ID 0 in its serialized parallel, and
+  // - there is only one thread per team
+  newTaskDescr->ThreadId() = 0;
+  newTaskDescr->ThreadsInTeam() = 1;
+
+  // set new task descriptor as top
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(threadId,
+                                                             newTaskDescr);
+}
+
+EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc,
+                                           uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_end_serialized_parallel\n");
+
+  if (checkRuntimeUninitialized(loc)) {
+    ASSERT0(LT_FUSSY, checkSPMDMode(loc),
+            "Expected SPMD mode with uninitialized runtime.");
+    __SYNCTHREADS();
+    if (GetThreadIdInBlock() == 0)
+      --parallelLevel;
+    __SYNCTHREADS();
+    return;
+  }
+
+  // pop stack
+  int threadId = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(threadId);
+  // set new top
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(
+      threadId, currTaskDescr->GetPrevTaskDescr());
+  // free
+  SafeFree(currTaskDescr, (char *)"new seq parallel task");
+  currTaskDescr = getMyTopTaskDescriptor(threadId);
+  currTaskDescr->RestoreLoopData();
+}
+
+EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_parallel_level\n");
+
+  if (checkRuntimeUninitialized(loc)) {
+    ASSERT0(LT_FUSSY, checkSPMDMode(loc),
+            "Expected SPMD mode with uninitialized runtime.");
+    return parallelLevel;
+  }
+
+  int threadId = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_TaskDescr *currTaskDescr =
+      omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+  if (currTaskDescr->InL2OrHigherParallelRegion())
+    return 2;
+  else if (currTaskDescr->InParallelRegion())
+    return 1;
+  else
+    return 0;
+}
+
+// This kmpc call returns the thread id across all teams. It's value is
+// cached by the compiler and used when calling the runtime. On nvptx
+// it's cheap to recalculate this value so we never use the result
+// of this call.
+EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc) {
+  int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  return GetOmpThreadId(tid, checkSPMDMode(loc),
+                        checkRuntimeUninitialized(loc));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// push params
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t tid,
+                                    int32_t num_threads) {
+  PRINT(LD_IO, "call kmpc_push_num_threads %d\n", num_threads);
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized.");
+  tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_threadPrivateContext->NumThreadsForNextParallel(tid) =
+      num_threads;
+}
+
+EXTERN void __kmpc_push_simd_limit(kmp_Ident *loc, int32_t tid,
+                                   int32_t simd_limit) {
+  PRINT(LD_IO, "call kmpc_push_simd_limit %d\n", (int)simd_limit);
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized.");
+  tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_threadPrivateContext->SimdLimitForNextSimd(tid) = simd_limit;
+}
+
+// Do nothing. The host guarantees we started the requested number of
+// teams and we only need inspection of gridDim.
+
+EXTERN void __kmpc_push_num_teams(kmp_Ident *loc, int32_t tid,
+                                  int32_t num_teams, int32_t thread_limit) {
+  PRINT(LD_IO, "call kmpc_push_num_teams %d\n", (int)num_teams);
+  ASSERT0(LT_FUSSY, FALSE,
+          "should never have anything with new teams on device");
+}
+
+EXTERN void __kmpc_push_proc_bind(kmp_Ident *loc, uint32_t tid,
+                                  int proc_bind) {
+  PRINT(LD_IO, "call kmpc_push_proc_bind %d\n", (int)proc_bind);
+}

libomptarget/deviceRTLs/nvptx/src/reduction.cu

@@ -0,0 +1,457 @@
+//===---- reduction.cu - NVPTX OpenMP reduction implementation ---- CUDA
+//-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of reduction with KMPC interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include <complex.h>
+#include <stdio.h>
+
+#include "omptarget-nvptx.h"
+
+// may eventually remove this
+EXTERN
+int32_t __gpu_block_reduce() {
+  bool isSPMDExecutionMode = isSPMDMode();
+  int tid = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
+  int nt =
+      GetNumberOfOmpThreads(tid, isSPMDExecutionMode, isRuntimeUninitialized());
+  if (nt != blockDim.x)
+    return 0;
+  unsigned tnum = __ACTIVEMASK();
+  if (tnum != (~0x0)) // assume swapSize is 32
+    return 0;
+  return 1;
+}
+
+EXTERN
+int32_t __kmpc_reduce_gpu(kmp_Ident *loc, int32_t global_tid, int32_t num_vars,
+                          size_t reduce_size, void *reduce_data,
+                          void *reduce_array_size, kmp_ReductFctPtr *reductFct,
+                          kmp_CriticalName *lck) {
+  int threadId = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(threadId);
+  int numthread;
+  if (currTaskDescr->IsParallelConstruct()) {
+    numthread =
+        GetNumberOfOmpThreads(threadId, checkSPMDMode(loc),
+                              checkRuntimeUninitialized(loc));
+  } else {
+    numthread = GetNumberOfOmpTeams();
+  }
+
+  if (numthread == 1)
+    return 1;
+  if (!__gpu_block_reduce())
+    return 2;
+  if (threadIdx.x == 0)
+    return 1;
+  return 0;
+}
+
+EXTERN
+int32_t __kmpc_reduce_combined(kmp_Ident *loc) {
+  return threadIdx.x == 0 ? 2 : 0;
+}
+
+EXTERN
+int32_t __kmpc_reduce_simd(kmp_Ident *loc) {
+  return (threadIdx.x % 32 == 0) ? 1 : 0;
+}
+
+EXTERN
+void __kmpc_nvptx_end_reduce(int32_t global_tid) {}
+
+EXTERN
+void __kmpc_nvptx_end_reduce_nowait(int32_t global_tid) {}
+
+EXTERN int32_t __kmpc_shuffle_int32(int32_t val, int16_t delta, int16_t size) {
+  return __SHFL_DOWN_SYNC(0xFFFFFFFF, val, delta, size);
+}
+
+EXTERN int64_t __kmpc_shuffle_int64(int64_t val, int16_t delta, int16_t size) {
+   int lo, hi;
+   asm volatile("mov.b64 {%0,%1}, %2;" : "=r"(lo), "=r"(hi) : "l"(val));
+   hi = __SHFL_DOWN_SYNC(0xFFFFFFFF, hi, delta, size);
+   lo = __SHFL_DOWN_SYNC(0xFFFFFFFF, lo, delta, size);
+   asm volatile("mov.b64 %0, {%1,%2};" : "=l"(val) : "r"(lo), "r"(hi));
+   return val;
+}
+
+INLINE static void gpu_regular_warp_reduce(void *reduce_data,
+                                           kmp_ShuffleReductFctPtr shflFct) {
+  for (uint32_t mask = WARPSIZE / 2; mask > 0; mask /= 2) {
+    shflFct(reduce_data, /*LaneId - not used= */ 0,
+            /*Offset = */ mask, /*AlgoVersion=*/0);
+  }
+}
+
+INLINE static void gpu_irregular_warp_reduce(void *reduce_data,
+                                             kmp_ShuffleReductFctPtr shflFct,
+                                             uint32_t size, uint32_t tid) {
+  uint32_t curr_size;
+  uint32_t mask;
+  curr_size = size;
+  mask = curr_size / 2;
+  while (mask > 0) {
+    shflFct(reduce_data, /*LaneId = */ tid, /*Offset=*/mask, /*AlgoVersion=*/1);
+    curr_size = (curr_size + 1) / 2;
+    mask = curr_size / 2;
+  }
+}
+
+INLINE static uint32_t
+gpu_irregular_simd_reduce(void *reduce_data, kmp_ShuffleReductFctPtr shflFct) {
+  uint32_t lanemask_lt;
+  uint32_t lanemask_gt;
+  uint32_t size, remote_id, physical_lane_id;
+  physical_lane_id = GetThreadIdInBlock() % WARPSIZE;
+  asm("mov.u32 %0, %%lanemask_lt;" : "=r"(lanemask_lt));
+  uint32_t Liveness = __ACTIVEMASK();
+  uint32_t logical_lane_id = __popc(Liveness & lanemask_lt) * 2;
+  asm("mov.u32 %0, %%lanemask_gt;" : "=r"(lanemask_gt));
+  do {
+    Liveness = __ACTIVEMASK();
+    remote_id = __ffs(Liveness & lanemask_gt);
+    size = __popc(Liveness);
+    logical_lane_id /= 2;
+    shflFct(reduce_data, /*LaneId =*/logical_lane_id,
+            /*Offset=*/remote_id - 1 - physical_lane_id, /*AlgoVersion=*/2);
+  } while (logical_lane_id % 2 == 0 && size > 1);
+  return (logical_lane_id == 0);
+}
+
+EXTERN
+int32_t __kmpc_nvptx_simd_reduce_nowait(int32_t global_tid, int32_t num_vars,
+                                        size_t reduce_size, void *reduce_data,
+                                        kmp_ShuffleReductFctPtr shflFct,
+                                        kmp_InterWarpCopyFctPtr cpyFct) {
+  uint32_t Liveness = __ACTIVEMASK();
+  if (Liveness == 0xffffffff) {
+    gpu_regular_warp_reduce(reduce_data, shflFct);
+    return GetThreadIdInBlock() % WARPSIZE ==
+           0; // Result on lane 0 of the simd warp.
+  } else {
+    return gpu_irregular_simd_reduce(
+        reduce_data, shflFct); // Result on the first active lane.
+  }
+}
+
+INLINE
+static int32_t nvptx_parallel_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    bool isSPMDExecutionMode, bool isRuntimeUninitialized) {
+  uint32_t BlockThreadId = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
+  uint32_t NumThreads = GetNumberOfOmpThreads(
+      BlockThreadId, isSPMDExecutionMode, isRuntimeUninitialized);
+  if (NumThreads == 1)
+    return 1;
+  /*
+   * This reduce function handles reduction within a team. It handles
+   * parallel regions in both L1 and L2 parallelism levels. It also
+   * supports Generic, SPMD, and NoOMP modes.
+   *
+   * 1. Reduce within a warp.
+   * 2. Warp master copies value to warp 0 via shared memory.
+   * 3. Warp 0 reduces to a single value.
+   * 4. The reduced value is available in the thread that returns 1.
+   */
+
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  uint32_t WarpsNeeded = (NumThreads + WARPSIZE - 1) / WARPSIZE;
+  uint32_t WarpId = BlockThreadId / WARPSIZE;
+
+  // Volta execution model:
+  // For the Generic execution mode a parallel region either has 1 thread and
+  // beyond that, always a multiple of 32. For the SPMD execution mode we may
+  // have any number of threads.
+  if ((NumThreads % WARPSIZE == 0) || (WarpId < WarpsNeeded - 1))
+    gpu_regular_warp_reduce(reduce_data, shflFct);
+  else if (NumThreads > 1) // Only SPMD execution mode comes thru this case.
+    gpu_irregular_warp_reduce(reduce_data, shflFct,
+                              /*LaneCount=*/NumThreads % WARPSIZE,
+                              /*LaneId=*/GetThreadIdInBlock() % WARPSIZE);
+
+  // When we have more than [warpsize] number of threads
+  // a block reduction is performed here.
+  //
+  // Only L1 parallel region can enter this if condition.
+  if (NumThreads > WARPSIZE) {
+    // Gather all the reduced values from each warp
+    // to the first warp.
+    cpyFct(reduce_data, WarpsNeeded);
+
+    if (WarpId == 0)
+      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded,
+                                BlockThreadId);
+  }
+  return BlockThreadId == 0;
+#else
+  uint32_t Liveness = __ACTIVEMASK();
+  if (Liveness == 0xffffffff) // Full warp
+    gpu_regular_warp_reduce(reduce_data, shflFct);
+  else if (!(Liveness & (Liveness + 1))) // Partial warp but contiguous lanes
+    gpu_irregular_warp_reduce(reduce_data, shflFct,
+                              /*LaneCount=*/__popc(Liveness),
+                              /*LaneId=*/GetThreadIdInBlock() % WARPSIZE);
+  else if (!isRuntimeUninitialized) // Dispersed lanes. Only threads in L2
+                                    // parallel region may enter here; return
+                                    // early.
+    return gpu_irregular_simd_reduce(reduce_data, shflFct);
+
+  // When we have more than [warpsize] number of threads
+  // a block reduction is performed here.
+  //
+  // Only L1 parallel region can enter this if condition.
+  if (NumThreads > WARPSIZE) {
+    uint32_t WarpsNeeded = (NumThreads + WARPSIZE - 1) / WARPSIZE;
+    // Gather all the reduced values from each warp
+    // to the first warp.
+    cpyFct(reduce_data, WarpsNeeded);
+
+    uint32_t WarpId = BlockThreadId / WARPSIZE;
+    if (WarpId == 0)
+      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded,
+                                BlockThreadId);
+
+    return BlockThreadId == 0;
+  } else if (isRuntimeUninitialized /* Never an L2 parallel region without the OMP runtime */) {
+    return BlockThreadId == 0;
+  }
+
+  // Get the OMP thread Id. This is different from BlockThreadId in the case of
+  // an L2 parallel region.
+  return global_tid == 0;
+#endif // __CUDA_ARCH__ >= 700
+}
+
+EXTERN __attribute__((deprecated)) int32_t __kmpc_nvptx_parallel_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
+  return nvptx_parallel_reduce_nowait(
+      global_tid, num_vars, reduce_size, reduce_data, shflFct, cpyFct,
+      /*isSPMDExecutionMode=*/isSPMDMode(),
+      /*isRuntimeUninitialized=*/isRuntimeUninitialized());
+}
+
+EXTERN
+int32_t __kmpc_nvptx_parallel_reduce_nowait_v2(
+    kmp_Ident *loc, int32_t global_tid, int32_t num_vars, size_t reduce_size,
+    void *reduce_data, kmp_ShuffleReductFctPtr shflFct,
+    kmp_InterWarpCopyFctPtr cpyFct) {
+  return nvptx_parallel_reduce_nowait(
+      global_tid, num_vars, reduce_size, reduce_data, shflFct, cpyFct,
+      checkSPMDMode(loc), checkRuntimeUninitialized(loc));
+}
+
+EXTERN
+int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_spmd(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
+  return nvptx_parallel_reduce_nowait(global_tid, num_vars, reduce_size,
+                                      reduce_data, shflFct, cpyFct,
+                                      /*isSPMDExecutionMode=*/true,
+                                      /*isRuntimeUninitialized=*/true);
+}
+
+EXTERN
+int32_t __kmpc_nvptx_parallel_reduce_nowait_simple_generic(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
+  return nvptx_parallel_reduce_nowait(global_tid, num_vars, reduce_size,
+                                      reduce_data, shflFct, cpyFct,
+                                      /*isSPMDExecutionMode=*/false,
+                                      /*isRuntimeUninitialized=*/true);
+}
+
+INLINE
+static int32_t nvptx_teams_reduce_nowait(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr scratchFct, kmp_LoadReduceFctPtr ldFct,
+    bool isSPMDExecutionMode, bool isRuntimeUninitialized) {
+  uint32_t ThreadId = GetLogicalThreadIdInBlock(isSPMDExecutionMode);
+  // In non-generic mode all workers participate in the teams reduction.
+  // In generic mode only the team master participates in the teams
+  // reduction because the workers are waiting for parallel work.
+  uint32_t NumThreads =
+      isSPMDExecutionMode
+          ? GetNumberOfOmpThreads(ThreadId, /*isSPMDExecutionMode=*/true,
+                                  isRuntimeUninitialized)
+          : /*Master thread only*/ 1;
+  uint32_t TeamId = GetBlockIdInKernel();
+  uint32_t NumTeams = GetNumberOfBlocksInKernel();
+  __shared__ volatile bool IsLastTeam;
+
+  // Team masters of all teams write to the scratchpad.
+  if (ThreadId == 0) {
+    unsigned int *timestamp = GetTeamsReductionTimestamp();
+    char *scratchpad = GetTeamsReductionScratchpad();
+
+    scratchFct(reduce_data, scratchpad, TeamId, NumTeams);
+    __threadfence();
+
+    // atomicInc increments 'timestamp' and has a range [0, NumTeams-1].
+    // It resets 'timestamp' back to 0 once the last team increments
+    // this counter.
+    unsigned val = atomicInc(timestamp, NumTeams - 1);
+    IsLastTeam = val == NumTeams - 1;
+  }
+
+  // We have to wait on L1 barrier because in GENERIC mode the workers
+  // are waiting on barrier 0 for work.
+  //
+  // If we guard this barrier as follows it leads to deadlock, probably
+  // because of a compiler bug: if (!IsGenericMode()) __syncthreads();
+  uint16_t SyncWarps = (NumThreads + WARPSIZE - 1) / WARPSIZE;
+  named_sync(L1_BARRIER, SyncWarps * WARPSIZE);
+
+  // If this team is not the last, quit.
+  if (/* Volatile read by all threads */ !IsLastTeam)
+    return 0;
+
+    //
+    // Last team processing.
+    //
+
+    // Threads in excess of #teams do not participate in reduction of the
+    // scratchpad values.
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  uint32_t ActiveThreads = NumThreads;
+  if (NumTeams < NumThreads) {
+    ActiveThreads =
+        (NumTeams < WARPSIZE) ? 1 : NumTeams & ~((uint16_t)WARPSIZE - 1);
+  }
+  if (ThreadId >= ActiveThreads)
+    return 0;
+
+  // Load from scratchpad and reduce.
+  char *scratchpad = GetTeamsReductionScratchpad();
+  ldFct(reduce_data, scratchpad, ThreadId, NumTeams, /*Load only*/ 0);
+  for (uint32_t i = ActiveThreads + ThreadId; i < NumTeams; i += ActiveThreads)
+    ldFct(reduce_data, scratchpad, i, NumTeams, /*Load and reduce*/ 1);
+
+  uint32_t WarpsNeeded = (ActiveThreads + WARPSIZE - 1) / WARPSIZE;
+  uint32_t WarpId = ThreadId / WARPSIZE;
+
+  // Reduce across warps to the warp master.
+  if ((ActiveThreads % WARPSIZE == 0) ||
+      (WarpId < WarpsNeeded - 1)) // Full warp
+    gpu_regular_warp_reduce(reduce_data, shflFct);
+  else if (ActiveThreads > 1) // Partial warp but contiguous lanes
+    // Only SPMD execution mode comes thru this case.
+    gpu_irregular_warp_reduce(reduce_data, shflFct,
+                              /*LaneCount=*/ActiveThreads % WARPSIZE,
+                              /*LaneId=*/ThreadId % WARPSIZE);
+
+  // When we have more than [warpsize] number of threads
+  // a block reduction is performed here.
+  if (ActiveThreads > WARPSIZE) {
+    // Gather all the reduced values from each warp
+    // to the first warp.
+    cpyFct(reduce_data, WarpsNeeded);
+
+    if (WarpId == 0)
+      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded, ThreadId);
+  }
+#else
+  if (ThreadId >= NumTeams)
+    return 0;
+
+  // Load from scratchpad and reduce.
+  char *scratchpad = GetTeamsReductionScratchpad();
+  ldFct(reduce_data, scratchpad, ThreadId, NumTeams, /*Load only*/ 0);
+  for (uint32_t i = NumThreads + ThreadId; i < NumTeams; i += NumThreads)
+    ldFct(reduce_data, scratchpad, i, NumTeams, /*Load and reduce*/ 1);
+
+  // Reduce across warps to the warp master.
+  uint32_t Liveness = __ACTIVEMASK();
+  if (Liveness == 0xffffffff) // Full warp
+    gpu_regular_warp_reduce(reduce_data, shflFct);
+  else // Partial warp but contiguous lanes
+    gpu_irregular_warp_reduce(reduce_data, shflFct,
+                              /*LaneCount=*/__popc(Liveness),
+                              /*LaneId=*/ThreadId % WARPSIZE);
+
+  // When we have more than [warpsize] number of threads
+  // a block reduction is performed here.
+  uint32_t ActiveThreads = NumTeams < NumThreads ? NumTeams : NumThreads;
+  if (ActiveThreads > WARPSIZE) {
+    uint32_t WarpsNeeded = (ActiveThreads + WARPSIZE - 1) / WARPSIZE;
+    // Gather all the reduced values from each warp
+    // to the first warp.
+    cpyFct(reduce_data, WarpsNeeded);
+
+    uint32_t WarpId = ThreadId / WARPSIZE;
+    if (WarpId == 0)
+      gpu_irregular_warp_reduce(reduce_data, shflFct, WarpsNeeded, ThreadId);
+  }
+#endif // __CUDA_ARCH__ >= 700
+
+  return ThreadId == 0;
+}
+
+EXTERN
+int32_t __kmpc_nvptx_teams_reduce_nowait(int32_t global_tid, int32_t num_vars,
+                                         size_t reduce_size, void *reduce_data,
+                                         kmp_ShuffleReductFctPtr shflFct,
+                                         kmp_InterWarpCopyFctPtr cpyFct,
+                                         kmp_CopyToScratchpadFctPtr scratchFct,
+                                         kmp_LoadReduceFctPtr ldFct) {
+  return nvptx_teams_reduce_nowait(
+      global_tid, num_vars, reduce_size, reduce_data, shflFct, cpyFct,
+      scratchFct, ldFct, /*isSPMDExecutionMode=*/isSPMDMode(),
+      /*isRuntimeUninitialized=*/isRuntimeUninitialized());
+}
+
+EXTERN
+int32_t __kmpc_nvptx_teams_reduce_nowait_simple_spmd(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr scratchFct, kmp_LoadReduceFctPtr ldFct) {
+  return nvptx_teams_reduce_nowait(global_tid, num_vars, reduce_size,
+                                   reduce_data, shflFct, cpyFct, scratchFct,
+                                   ldFct,
+                                   /*isSPMDExecutionMode=*/true,
+                                   /*isRuntimeUninitialized=*/true);
+}
+
+EXTERN
+int32_t __kmpc_nvptx_teams_reduce_nowait_simple_generic(
+    int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
+    kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct,
+    kmp_CopyToScratchpadFctPtr scratchFct, kmp_LoadReduceFctPtr ldFct) {
+  return nvptx_teams_reduce_nowait(global_tid, num_vars, reduce_size,
+                                   reduce_data, shflFct, cpyFct, scratchFct,
+                                   ldFct,
+                                   /*isSPMDExecutionMode=*/false,
+                                   /*isRuntimeUninitialized=*/true);
+}
+
+EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_simple(kmp_Ident *loc,
+                                                       int32_t global_tid,
+                                                       kmp_CriticalName *crit) {
+  if (checkSPMDMode(loc) && GetThreadIdInBlock() != 0)
+    return 0;
+  // The master thread of the team actually does the reduction.
+  while (atomicCAS((uint32_t *)crit, 0, 1))
+    ;
+  return 1;
+}
+
+EXTERN void
+__kmpc_nvptx_teams_end_reduce_nowait_simple(kmp_Ident *loc, int32_t global_tid,
+                                            kmp_CriticalName *crit) {
+  __threadfence_system();
+  (void)atomicExch((uint32_t *)crit, 0);
+}
+

libomptarget/deviceRTLs/nvptx/src/state-queue.h

@@ -0,0 +1,52 @@
+//===--------- statequeue.h - NVPTX OpenMP GPU State Queue ------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a queue to hand out OpenMP state objects to teams of
+// one or more kernels.
+//
+// Reference:
+// Thomas R.W. Scogland and Wu-chun Feng. 2015.
+// Design and Evaluation of Scalable Concurrent Queues for Many-Core
+// Architectures. International Conference on Performance Engineering.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __STATE_QUEUE_H
+#define __STATE_QUEUE_H
+
+#include <stdint.h>
+
+#include "option.h" // choices we have
+
+template <typename ElementType, uint32_t SIZE> class omptarget_nvptx_Queue {
+private:
+  ElementType elements[SIZE];
+  volatile ElementType *elementQueue[SIZE];
+  volatile uint32_t head;
+  volatile uint32_t ids[SIZE];
+  volatile uint32_t tail;
+
+  static const uint32_t MAX_ID = (1u << 31) / SIZE / 2;
+  INLINE uint32_t ENQUEUE_TICKET();
+  INLINE uint32_t DEQUEUE_TICKET();
+  INLINE static uint32_t ID(uint32_t ticket);
+  INLINE bool IsServing(uint32_t slot, uint32_t id);
+  INLINE void PushElement(uint32_t slot, ElementType *element);
+  INLINE ElementType *PopElement(uint32_t slot);
+  INLINE void DoneServing(uint32_t slot, uint32_t id);
+
+public:
+  INLINE omptarget_nvptx_Queue() {}
+  INLINE void Enqueue(ElementType *element);
+  INLINE ElementType *Dequeue();
+};
+
+#include "state-queuei.h"
+
+#endif

libomptarget/deviceRTLs/nvptx/src/state-queuei.h

@@ -0,0 +1,90 @@
+//===------- state-queue.cu - NVPTX OpenMP GPU State Queue ------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the implementation of a queue to hand out OpenMP state
+// objects to teams of one or more kernels.
+//
+// Reference:
+// Thomas R.W. Scogland and Wu-chun Feng. 2015.
+// Design and Evaluation of Scalable Concurrent Queues for Many-Core
+// Architectures. International Conference on Performance Engineering.
+//
+//===----------------------------------------------------------------------===//
+
+#include "state-queue.h"
+
+template <typename ElementType, uint32_t SIZE>
+INLINE uint32_t omptarget_nvptx_Queue<ElementType, SIZE>::ENQUEUE_TICKET() {
+  return atomicAdd((unsigned int *)&tail, 1);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE uint32_t omptarget_nvptx_Queue<ElementType, SIZE>::DEQUEUE_TICKET() {
+  return atomicAdd((unsigned int *)&head, 1);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE uint32_t
+omptarget_nvptx_Queue<ElementType, SIZE>::ID(uint32_t ticket) {
+  return (ticket / SIZE) * 2;
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE bool omptarget_nvptx_Queue<ElementType, SIZE>::IsServing(uint32_t slot,
+                                                                uint32_t id) {
+  return atomicAdd((unsigned int *)&ids[slot], 0) == id;
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE void
+omptarget_nvptx_Queue<ElementType, SIZE>::PushElement(uint32_t slot,
+                                                      ElementType *element) {
+  atomicExch((unsigned long long *)&elementQueue[slot],
+             (unsigned long long)element);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE ElementType *
+omptarget_nvptx_Queue<ElementType, SIZE>::PopElement(uint32_t slot) {
+  return (ElementType *)atomicAdd((unsigned long long *)&elementQueue[slot],
+                                  (unsigned long long)0);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE void omptarget_nvptx_Queue<ElementType, SIZE>::DoneServing(uint32_t slot,
+                                                                  uint32_t id) {
+  atomicExch((unsigned int *)&ids[slot], (id + 1) % MAX_ID);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE void
+omptarget_nvptx_Queue<ElementType, SIZE>::Enqueue(ElementType *element) {
+  uint32_t ticket = ENQUEUE_TICKET();
+  uint32_t slot = ticket % SIZE;
+  uint32_t id = ID(ticket) + 1;
+  while (!IsServing(slot, id))
+    ;
+  PushElement(slot, element);
+  DoneServing(slot, id);
+}
+
+template <typename ElementType, uint32_t SIZE>
+INLINE ElementType *omptarget_nvptx_Queue<ElementType, SIZE>::Dequeue() {
+  uint32_t ticket = DEQUEUE_TICKET();
+  uint32_t slot = ticket % SIZE;
+  uint32_t id = ID(ticket);
+  while (!IsServing(slot, id))
+    ;
+  ElementType *element = PopElement(slot);
+  // This is to populate the queue because of the lack of GPU constructors.
+  if (element == 0)
+    element = &elements[slot];
+  DoneServing(slot, id);
+  return element;
+}

libomptarget/deviceRTLs/nvptx/src/support.h

@@ -0,0 +1,92 @@
+//===--------- support.h - NVPTX OpenMP support functions -------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Wrapper to some functions natively supported by the GPU.
+//
+//===----------------------------------------------------------------------===//
+
+////////////////////////////////////////////////////////////////////////////////
+// Execution Parameters
+////////////////////////////////////////////////////////////////////////////////
+enum ExecutionMode {
+  Generic = 0x00u,
+  Spmd = 0x01u,
+  ModeMask = 0x01u,
+};
+
+enum RuntimeMode {
+  RuntimeInitialized = 0x00u,
+  RuntimeUninitialized = 0x02u,
+  RuntimeMask = 0x02u,
+};
+
+INLINE void setExecutionParameters(ExecutionMode EMode, RuntimeMode RMode);
+INLINE bool isGenericMode();
+INLINE bool isSPMDMode();
+INLINE bool isRuntimeUninitialized();
+INLINE bool isRuntimeInitialized();
+
+////////////////////////////////////////////////////////////////////////////////
+// get info from machine
+////////////////////////////////////////////////////////////////////////////////
+
+// get low level ids of resources
+INLINE int GetThreadIdInBlock();
+INLINE int GetBlockIdInKernel();
+INLINE int GetNumberOfBlocksInKernel();
+INLINE int GetNumberOfThreadsInBlock();
+
+// get global ids to locate tread/team info (constant regardless of OMP)
+INLINE int GetLogicalThreadIdInBlock(bool isSPMDExecutionMode);
+INLINE int GetMasterThreadID();
+INLINE int GetNumberOfWorkersInTeam();
+
+// get OpenMP thread and team ids
+INLINE int GetOmpThreadId(int threadId, bool isSPMDExecutionMode,
+                          bool isRuntimeUninitialized); // omp_thread_num
+INLINE int GetOmpTeamId();                              // omp_team_num
+
+// get OpenMP number of threads and team
+INLINE int
+GetNumberOfOmpThreads(int threadId, bool isSPMDExecutionMode,
+                      bool isRuntimeUninitialized); // omp_num_threads
+INLINE int GetNumberOfOmpTeams();                   // omp_num_teams
+
+// get OpenMP number of procs
+INLINE int GetNumberOfProcsInTeam(bool isSPMDExecutionMode);
+INLINE int GetNumberOfProcsInDevice(bool isSPMDExecutionMode);
+
+// masters
+INLINE int IsTeamMaster(int ompThreadId);
+
+////////////////////////////////////////////////////////////////////////////////
+// Memory
+////////////////////////////////////////////////////////////////////////////////
+
+// safe alloc and free
+INLINE void *SafeMalloc(size_t size, const char *msg); // check if success
+INLINE void *SafeFree(void *ptr, const char *msg);
+// pad to a alignment (power of 2 only)
+INLINE unsigned long PadBytes(unsigned long size, unsigned long alignment);
+#define ADD_BYTES(_addr, _bytes)                                               \
+  ((void *)((char *)((void *)(_addr)) + (_bytes)))
+#define SUB_BYTES(_addr, _bytes)                                               \
+  ((void *)((char *)((void *)(_addr)) - (_bytes)))
+
+////////////////////////////////////////////////////////////////////////////////
+// Named Barrier Routines
+////////////////////////////////////////////////////////////////////////////////
+INLINE void named_sync(const int barrier, const int num_threads);
+
+////////////////////////////////////////////////////////////////////////////////
+// Teams Reduction Scratchpad Helpers
+////////////////////////////////////////////////////////////////////////////////
+INLINE unsigned int *GetTeamsReductionTimestamp();
+INLINE char *GetTeamsReductionScratchpad();
+INLINE void SetTeamsReductionScratchpadPtr(void *ScratchpadPtr);

libomptarget/deviceRTLs/nvptx/src/supporti.h

@@ -0,0 +1,277 @@
+//===--------- supporti.h - NVPTX OpenMP support functions ------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Wrapper implementation to some functions natively supported by the GPU.
+//
+//===----------------------------------------------------------------------===//
+
+////////////////////////////////////////////////////////////////////////////////
+// Execution Parameters
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE void setExecutionParameters(ExecutionMode EMode, RuntimeMode RMode) {
+  execution_param = EMode;
+  execution_param |= RMode;
+}
+
+INLINE bool isGenericMode() { return (execution_param & ModeMask) == Generic; }
+
+INLINE bool isSPMDMode() { return (execution_param & ModeMask) == Spmd; }
+
+INLINE bool isRuntimeUninitialized() {
+  return (execution_param & RuntimeMask) == RuntimeUninitialized;
+}
+
+INLINE bool isRuntimeInitialized() {
+  return (execution_param & RuntimeMask) == RuntimeInitialized;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Execution Modes based on location parameter fields
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE bool checkSPMDMode(kmp_Ident *loc) {
+  if (!loc)
+    return isSPMDMode();
+
+  // If SPMD is true then we are not in the UNDEFINED state so
+  // we can return immediately.
+  if (loc->reserved_2 & KMP_IDENT_SPMD_MODE)
+    return true;
+
+  // If not in SPMD mode and runtime required is a valid
+  // combination of flags so we can return immediately.
+  if (!(loc->reserved_2 & KMP_IDENT_SIMPLE_RT_MODE))
+    return false;
+
+  // We are in underfined state.
+  return isSPMDMode();
+}
+
+INLINE bool checkGenericMode(kmp_Ident *loc) {
+  return !checkSPMDMode(loc);
+}
+
+INLINE bool checkRuntimeUninitialized(kmp_Ident *loc) {
+  if (!loc)
+    return isRuntimeUninitialized();
+
+  // If runtime is required then we know we can't be
+  // in the undefined mode. We can return immediately.
+  if (!(loc->reserved_2 & KMP_IDENT_SIMPLE_RT_MODE))
+    return false;
+
+  // If runtime is required then we need to check is in
+  // SPMD mode or not. If not in SPMD mode then we end
+  // up in the UNDEFINED state that marks the orphaned
+  // functions.
+  if (loc->reserved_2 & KMP_IDENT_SPMD_MODE)
+    return true;
+
+  // Check if we are in an UNDEFINED state. Undefined is denoted by
+  // non-SPMD + noRuntimeRequired which is a combination that
+  // cannot actually happen. Undefined states is used to mark orphaned
+  // functions.
+  return isRuntimeUninitialized();
+}
+
+INLINE bool checkRuntimeInitialized(kmp_Ident *loc) {
+  return !checkRuntimeUninitialized(loc);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// support: get info from machine
+////////////////////////////////////////////////////////////////////////////////
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// Calls to the NVPTX layer  (assuming 1D layout)
+//
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE int GetThreadIdInBlock() { return threadIdx.x; }
+
+INLINE int GetBlockIdInKernel() { return blockIdx.x; }
+
+INLINE int GetNumberOfBlocksInKernel() { return gridDim.x; }
+
+INLINE int GetNumberOfThreadsInBlock() { return blockDim.x; }
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// Calls to the Generic Scheme Implementation Layer (assuming 1D layout)
+//
+////////////////////////////////////////////////////////////////////////////////
+
+// The master thread id is the first thread (lane) of the last warp.
+// Thread id is 0 indexed.
+// E.g: If NumThreads is 33, master id is 32.
+//      If NumThreads is 64, master id is 32.
+//      If NumThreads is 97, master id is 96.
+//      If NumThreads is 1024, master id is 992.
+//
+// Called in Generic Execution Mode only.
+INLINE int GetMasterThreadID() { return (blockDim.x - 1) & ~(WARPSIZE - 1); }
+
+// The last warp is reserved for the master; other warps are workers.
+// Called in Generic Execution Mode only.
+INLINE int GetNumberOfWorkersInTeam() { return GetMasterThreadID(); }
+
+////////////////////////////////////////////////////////////////////////////////
+// get thread id in team
+
+// This function may be called in a parallel region by the workers
+// or a serial region by the master.  If the master (whose CUDA thread
+// id is GetMasterThreadID()) calls this routine, we return 0 because
+// it is a shadow for the first worker.
+INLINE int GetLogicalThreadIdInBlock(bool isSPMDExecutionMode) {
+  // Implemented using control flow (predication) instead of with a modulo
+  // operation.
+  int tid = GetThreadIdInBlock();
+  if (!isSPMDExecutionMode && tid >= GetMasterThreadID())
+    return 0;
+  else
+    return tid;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+// OpenMP Thread Support Layer
+//
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE int GetOmpThreadId(int threadId, bool isSPMDExecutionMode,
+                          bool isRuntimeUninitialized) {
+  // omp_thread_num
+  int rc;
+
+  if (isRuntimeUninitialized) {
+    ASSERT0(LT_FUSSY, isSPMDExecutionMode,
+            "Uninitialized runtime with non-SPMD mode.");
+    // For level 2 parallelism all parallel regions are executed sequentially.
+    if (parallelLevel > 0)
+      rc = 0;
+    else
+      rc = GetThreadIdInBlock();
+  } else {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+    rc = currTaskDescr->ThreadId();
+  }
+  return rc;
+}
+
+INLINE int GetNumberOfOmpThreads(int threadId, bool isSPMDExecutionMode,
+                                 bool isRuntimeUninitialized) {
+  // omp_num_threads
+  int rc;
+
+  if (isRuntimeUninitialized) {
+    ASSERT0(LT_FUSSY, isSPMDExecutionMode,
+            "Uninitialized runtime with non-SPMD mode.");
+    // For level 2 parallelism all parallel regions are executed sequentially.
+    if (parallelLevel > 0)
+      rc = 1;
+    else
+      rc = GetNumberOfThreadsInBlock();
+  } else {
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(threadId);
+    ASSERT0(LT_FUSSY, currTaskDescr, "expected a top task descr");
+    rc = currTaskDescr->ThreadsInTeam();
+  }
+
+  return rc;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Team id linked to OpenMP
+
+INLINE int GetOmpTeamId() {
+  // omp_team_num
+  return GetBlockIdInKernel(); // assume 1 block per team
+}
+
+INLINE int GetNumberOfOmpTeams() {
+  // omp_num_teams
+  return GetNumberOfBlocksInKernel(); // assume 1 block per team
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Masters
+
+INLINE int IsTeamMaster(int ompThreadId) { return (ompThreadId == 0); }
+
+////////////////////////////////////////////////////////////////////////////////
+// get OpenMP number of procs
+
+// Get the number of processors in the device.
+INLINE int GetNumberOfProcsInDevice(bool isSPMDExecutionMode) {
+  if (!isSPMDExecutionMode)
+    return GetNumberOfWorkersInTeam();
+  return GetNumberOfThreadsInBlock();
+}
+
+INLINE int GetNumberOfProcsInTeam(bool isSPMDExecutionMode) {
+  return GetNumberOfProcsInDevice(isSPMDExecutionMode);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Memory
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE unsigned long PadBytes(unsigned long size,
+                              unsigned long alignment) // must be a power of 2
+{
+  // compute the necessary padding to satisfy alignment constraint
+  ASSERT(LT_FUSSY, (alignment & (alignment - 1)) == 0,
+         "alignment %lu is not a power of 2\n", alignment);
+  return (~(unsigned long)size + 1) & (alignment - 1);
+}
+
+INLINE void *SafeMalloc(size_t size, const char *msg) // check if success
+{
+  void *ptr = malloc(size);
+  PRINT(LD_MEM, "malloc data of size %llu for %s: 0x%llx\n",
+        (unsigned long long)size, msg, (unsigned long long)ptr);
+  return ptr;
+}
+
+INLINE void *SafeFree(void *ptr, const char *msg) {
+  PRINT(LD_MEM, "free data ptr 0x%llx for %s\n", (unsigned long long)ptr, msg);
+  free(ptr);
+  return NULL;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Named Barrier Routines
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE void named_sync(const int barrier, const int num_threads) {
+  asm volatile("bar.sync %0, %1;"
+               :
+               : "r"(barrier), "r"(num_threads)
+               : "memory");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Teams Reduction Scratchpad Helpers
+////////////////////////////////////////////////////////////////////////////////
+
+INLINE unsigned int *GetTeamsReductionTimestamp() {
+  return static_cast<unsigned int *>(ReductionScratchpadPtr);
+}
+
+INLINE char *GetTeamsReductionScratchpad() {
+  return static_cast<char *>(ReductionScratchpadPtr) + 256;
+}
+
+INLINE void SetTeamsReductionScratchpadPtr(void *ScratchpadPtr) {
+  ReductionScratchpadPtr = ScratchpadPtr;
+}

libomptarget/deviceRTLs/nvptx/src/sync.cu

@@ -0,0 +1,146 @@
+//===------------ sync.h - NVPTX OpenMP synchronizations --------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Include all synchronization.
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP Ordered calls
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN void __kmpc_ordered(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_ordered\n");
+}
+
+EXTERN void __kmpc_end_ordered(kmp_Ident *loc, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_end_ordered\n");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP Barriers
+////////////////////////////////////////////////////////////////////////////////
+
+// a team is a block: we can use CUDA native synchronization mechanism
+// FIXME: what if not all threads (warps) participate to the barrier?
+// We may need to implement it differently
+
+EXTERN int32_t __kmpc_cancel_barrier(kmp_Ident *loc_ref, int32_t tid) {
+  PRINT0(LD_IO, "call kmpc_cancel_barrier\n");
+  __kmpc_barrier(loc_ref, tid);
+  PRINT0(LD_SYNC, "completed kmpc_cancel_barrier\n");
+  return 0;
+}
+
+EXTERN void __kmpc_barrier(kmp_Ident *loc_ref, int32_t tid) {
+  if (checkRuntimeUninitialized(loc_ref)) {
+    ASSERT0(LT_FUSSY, checkSPMDMode(loc_ref),
+            "Expected SPMD mode with uninitialized runtime.");
+    __kmpc_barrier_simple_spmd(loc_ref, tid);
+  } else {
+    tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc_ref));
+    omptarget_nvptx_TaskDescr *currTaskDescr =
+        omptarget_nvptx_threadPrivateContext->GetTopLevelTaskDescr(tid);
+    int numberOfActiveOMPThreads = GetNumberOfOmpThreads(
+        tid, checkSPMDMode(loc_ref), /*isRuntimeUninitialized=*/false);
+    if (numberOfActiveOMPThreads > 1) {
+      if (checkSPMDMode(loc_ref)) {
+        __kmpc_barrier_simple_spmd(loc_ref, tid);
+      } else {
+        // The #threads parameter must be rounded up to the WARPSIZE.
+        int threads =
+            WARPSIZE * ((numberOfActiveOMPThreads + WARPSIZE - 1) / WARPSIZE);
+
+        PRINT(LD_SYNC,
+              "call kmpc_barrier with %d omp threads, sync parameter %d\n",
+              (int)numberOfActiveOMPThreads, (int)threads);
+        // Barrier #1 is for synchronization among active threads.
+        named_sync(L1_BARRIER, threads);
+      }
+    } // numberOfActiveOMPThreads > 1
+    PRINT0(LD_SYNC, "completed kmpc_barrier\n");
+  }
+}
+
+// Emit a simple barrier call in SPMD mode.  Assumes the caller is in an L0
+// parallel region and that all worker threads participate.
+EXTERN void __kmpc_barrier_simple_spmd(kmp_Ident *loc_ref, int32_t tid) {
+  PRINT0(LD_SYNC, "call kmpc_barrier_simple_spmd\n");
+  // FIXME: use __syncthreads instead when the function copy is fixed in LLVM.
+  __SYNCTHREADS();
+  PRINT0(LD_SYNC, "completed kmpc_barrier_simple_spmd\n");
+}
+
+// Emit a simple barrier call in Generic mode.  Assumes the caller is in an L0
+// parallel region and that all worker threads participate.
+EXTERN void __kmpc_barrier_simple_generic(kmp_Ident *loc_ref, int32_t tid) {
+  int numberOfActiveOMPThreads = GetNumberOfThreadsInBlock() - WARPSIZE;
+  // The #threads parameter must be rounded up to the WARPSIZE.
+  int threads =
+      WARPSIZE * ((numberOfActiveOMPThreads + WARPSIZE - 1) / WARPSIZE);
+
+  PRINT(LD_SYNC,
+        "call kmpc_barrier_simple_generic with %d omp threads, sync parameter "
+        "%d\n",
+        (int)numberOfActiveOMPThreads, (int)threads);
+  // Barrier #1 is for synchronization among active threads.
+  named_sync(L1_BARRIER, threads);
+  PRINT0(LD_SYNC, "completed kmpc_barrier_simple_generic\n");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP MASTER
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN int32_t __kmpc_master(kmp_Ident *loc, int32_t global_tid) {
+  PRINT0(LD_IO, "call kmpc_master\n");
+  return IsTeamMaster(global_tid);
+}
+
+EXTERN void __kmpc_end_master(kmp_Ident *loc, int32_t global_tid) {
+  PRINT0(LD_IO, "call kmpc_end_master\n");
+  ASSERT0(LT_FUSSY, IsTeamMaster(global_tid), "expected only master here");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// KMP SINGLE
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN int32_t __kmpc_single(kmp_Ident *loc, int32_t global_tid) {
+  PRINT0(LD_IO, "call kmpc_single\n");
+  // decide to implement single with master; master get the single
+  return IsTeamMaster(global_tid);
+}
+
+EXTERN void __kmpc_end_single(kmp_Ident *loc, int32_t global_tid) {
+  PRINT0(LD_IO, "call kmpc_end_single\n");
+  // decide to implement single with master: master get the single
+  ASSERT0(LT_FUSSY, IsTeamMaster(global_tid), "expected only master here");
+  // sync barrier is explicitely called... so that is not a problem
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Flush
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN void __kmpc_flush(kmp_Ident *loc) {
+  PRINT0(LD_IO, "call kmpc_flush\n");
+  __threadfence_system();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Vote
+////////////////////////////////////////////////////////////////////////////////
+
+EXTERN int32_t __kmpc_warp_active_thread_mask() {
+  PRINT0(LD_IO, "call __kmpc_warp_active_thread_mask\n");
+  return __ACTIVEMASK();
+}

libomptarget/deviceRTLs/nvptx/src/task.cu

@@ -0,0 +1,217 @@
+//===------------- task.h - NVPTX OpenMP tasks support ----------- CUDA -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Task implementation support.
+//
+//  explicit task structure uses
+//  omptarget_nvptx task
+//  kmp_task
+//
+//  where kmp_task is
+//    - klegacy_TaskDescr    <- task pointer
+//        shared -> X
+//        routine
+//        part_id
+//        descr
+//    -  private (of size given by task_alloc call). Accessed by
+//       task+sizeof(klegacy_TaskDescr)
+//        * private data *
+//    - shared: X. Accessed by shared ptr in klegacy_TaskDescr
+//        * pointer table to shared variables *
+//    - end
+//
+//===----------------------------------------------------------------------===//
+
+#include "omptarget-nvptx.h"
+
+EXTERN kmp_TaskDescr *__kmpc_omp_task_alloc(
+    kmp_Ident *loc,     // unused
+    uint32_t global_tid, // unused
+    int32_t flag, // unused (because in our impl, all are immediately exec
+    size_t sizeOfTaskInclPrivate, size_t sizeOfSharedTable,
+    kmp_TaskFctPtr taskSub) {
+  PRINT(LD_IO,
+        "call __kmpc_omp_task_alloc(size priv&struct %lld, shared %lld, "
+        "fct 0x%llx)\n",
+        (long long)sizeOfTaskInclPrivate, (long long)sizeOfSharedTable,
+        (unsigned long long)taskSub);
+  // want task+priv to be a multiple of 8 bytes
+  size_t padForTaskInclPriv = PadBytes(sizeOfTaskInclPrivate, sizeof(void *));
+  sizeOfTaskInclPrivate += padForTaskInclPriv;
+  size_t kmpSize = sizeOfTaskInclPrivate + sizeOfSharedTable;
+  ASSERT(LT_FUSSY, sizeof(omptarget_nvptx_TaskDescr) % sizeof(void *) == 0,
+         "need task descr of size %d to be a multiple of %d\n",
+         (int)sizeof(omptarget_nvptx_TaskDescr), (int)sizeof(void *));
+  size_t totSize = sizeof(omptarget_nvptx_TaskDescr) + kmpSize;
+  omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
+      (omptarget_nvptx_ExplicitTaskDescr *)SafeMalloc(
+          totSize, "explicit task descriptor");
+  kmp_TaskDescr *newKmpTaskDescr = &newExplicitTaskDescr->kmpTaskDescr;
+  ASSERT0(LT_FUSSY,
+          (uint64_t)newKmpTaskDescr ==
+              (uint64_t)ADD_BYTES(newExplicitTaskDescr,
+                                  sizeof(omptarget_nvptx_TaskDescr)),
+          "bad size assumptions");
+  // init kmp_TaskDescr
+  newKmpTaskDescr->sharedPointerTable =
+      (void *)((char *)newKmpTaskDescr + sizeOfTaskInclPrivate);
+  newKmpTaskDescr->sub = taskSub;
+  newKmpTaskDescr->destructors = NULL;
+  PRINT(LD_TASK, "return with task descr kmp: 0x%llx, omptarget-nvptx 0x%llx\n",
+        (unsigned long long)newKmpTaskDescr,
+        (unsigned long long)newExplicitTaskDescr);
+
+  return newKmpTaskDescr;
+}
+
+EXTERN int32_t __kmpc_omp_task(kmp_Ident *loc, uint32_t global_tid,
+                               kmp_TaskDescr *newKmpTaskDescr) {
+  return __kmpc_omp_task_with_deps(loc, global_tid, newKmpTaskDescr, 0, 0, 0,
+                                   0);
+}
+
+EXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid,
+                                         kmp_TaskDescr *newKmpTaskDescr,
+                                         int32_t depNum, void *depList,
+                                         int32_t noAliasDepNum,
+                                         void *noAliasDepList) {
+  PRINT(LD_IO, "call to __kmpc_omp_task_with_deps(task 0x%llx)\n",
+        P64(newKmpTaskDescr));
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+          "Runtime must be initialized.");
+  // 1. get explict task descr from kmp task descr
+  omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
+      (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
+          newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
+  ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
+          "bad assumptions");
+  omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
+  ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
+          "bad assumptions");
+
+  // 2. push new context: update new task descriptor
+  int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_TaskDescr *parentTaskDescr = getMyTopTaskDescriptor(tid);
+  newTaskDescr->CopyForExplicitTask(parentTaskDescr);
+  // set new task descriptor as top
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid, newTaskDescr);
+
+  // 3. call sub
+  PRINT(LD_TASK, "call task sub 0x%llx(task descr 0x%llx)\n",
+        (unsigned long long)newKmpTaskDescr->sub,
+        (unsigned long long)newKmpTaskDescr);
+  newKmpTaskDescr->sub(0, newKmpTaskDescr);
+  PRINT(LD_TASK, "return from call task sub 0x%llx()\n",
+        (unsigned long long)newKmpTaskDescr->sub);
+
+  // 4. pop context
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid,
+                                                             parentTaskDescr);
+  // 5. free
+  SafeFree(newExplicitTaskDescr, "explicit task descriptor");
+  return 0;
+}
+
+EXTERN void __kmpc_omp_task_begin_if0(kmp_Ident *loc, uint32_t global_tid,
+                                      kmp_TaskDescr *newKmpTaskDescr) {
+  PRINT(LD_IO, "call to __kmpc_omp_task_begin_if0(task 0x%llx)\n",
+        (unsigned long long)newKmpTaskDescr);
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+          "Runtime must be initialized.");
+  // 1. get explict task descr from kmp task descr
+  omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
+      (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
+          newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
+  ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
+          "bad assumptions");
+  omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
+  ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
+          "bad assumptions");
+
+  // 2. push new context: update new task descriptor
+  int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_TaskDescr *parentTaskDescr = getMyTopTaskDescriptor(tid);
+  newTaskDescr->CopyForExplicitTask(parentTaskDescr);
+  // set new task descriptor as top
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid, newTaskDescr);
+  // 3... noting to call... is inline
+  // 4 & 5 ... done in complete
+}
+
+EXTERN void __kmpc_omp_task_complete_if0(kmp_Ident *loc, uint32_t global_tid,
+                                         kmp_TaskDescr *newKmpTaskDescr) {
+  PRINT(LD_IO, "call to __kmpc_omp_task_complete_if0(task 0x%llx)\n",
+        (unsigned long long)newKmpTaskDescr);
+  ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
+          "Runtime must be initialized.");
+  // 1. get explict task descr from kmp task descr
+  omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
+      (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
+          newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
+  ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
+          "bad assumptions");
+  omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
+  ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
+          "bad assumptions");
+  // 2. get parent
+  omptarget_nvptx_TaskDescr *parentTaskDescr = newTaskDescr->GetPrevTaskDescr();
+  // 3... noting to call... is inline
+  // 4. pop context
+  int tid = GetLogicalThreadIdInBlock(checkSPMDMode(loc));
+  omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid,
+                                                             parentTaskDescr);
+  // 5. free
+  SafeFree(newExplicitTaskDescr, "explicit task descriptor");
+}
+
+EXTERN void __kmpc_omp_wait_deps(kmp_Ident *loc, uint32_t global_tid,
+                                 int32_t depNum, void *depList,
+                                 int32_t noAliasDepNum, void *noAliasDepList) {
+  PRINT0(LD_IO, "call to __kmpc_omp_wait_deps(..)\n");
+  // nothing to do as all our tasks are executed as final
+}
+
+EXTERN void __kmpc_taskgroup(kmp_Ident *loc, uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_taskgroup(..)\n");
+  // nothing to do as all our tasks are executed as final
+}
+
+EXTERN void __kmpc_end_taskgroup(kmp_Ident *loc, uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_end_taskgroup(..)\n");
+  // nothing to do as all our tasks are executed as final
+}
+
+EXTERN int32_t __kmpc_omp_taskyield(kmp_Ident *loc, uint32_t global_tid,
+                                    int end_part) {
+  PRINT0(LD_IO, "call to __kmpc_taskyield()\n");
+  // do nothing: tasks are executed immediately, no yielding allowed
+  return 0;
+}
+
+EXTERN int32_t __kmpc_omp_taskwait(kmp_Ident *loc, uint32_t global_tid) {
+  PRINT0(LD_IO, "call to __kmpc_taskwait()\n");
+  // nothing to do as all our tasks are executed as final
+  return 0;
+}
+
+EXTERN void __kmpc_taskloop(kmp_Ident *loc, uint32_t global_tid,
+                            kmp_TaskDescr *newKmpTaskDescr, int if_val,
+                            uint64_t *lb, uint64_t *ub, int64_t st, int nogroup,
+                            int32_t sched, uint64_t grainsize, void *task_dup) {
+
+  // skip task entirely if empty iteration space
+  if (*lb > *ub)
+    return;
+
+  // the compiler has already stored lb and ub in the kmp_TaskDescr structure
+  // as we are using a single task to execute the entire loop, we can leave
+  // the initial task_t untouched
+
+  __kmpc_omp_task_with_deps(loc, global_tid, newKmpTaskDescr, 0, 0, 0, 0);
+}

libomptarget/deviceRTLs/nvptx/test/CMakeLists.txt

@@ -0,0 +1,26 @@
+if(NOT OPENMP_TEST_COMPILER_ID STREQUAL "Clang")
+  # Silently return, no need to annoy the user.
+  return()
+endif()
+
+set(deps omptarget-nvptx omptarget omp)
+if(LIBOMPTARGET_NVPTX_ENABLE_BCLIB)
+  set(deps ${deps} omptarget-nvptx-bc)
+endif()
+
+# Don't run by default.
+set(EXCLUDE_FROM_ALL True)
+# Run with only one thread to only launch one application to the GPU at a time.
+add_openmp_testsuite(check-libomptarget-nvptx
+    "Running libomptarget-nvptx tests" ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${deps} ARGS -j1)
+
+set(LIBOMPTARGET_NVPTX_TEST_FLAGS "" CACHE STRING
+    "Extra compiler flags to send to the test compiler.")
+set(LIBOMPTARGET_NVPTX_TEST_OPENMP_FLAGS
+    "-fopenmp -fopenmp-targets=nvptx64-nvidia-cuda" CACHE STRING
+    "OpenMP compiler flags to use for testing libomptarget-nvptx.")
+
+# Configure the lit.site.cfg.in file
+set(AUTO_GEN_COMMENT "## Autogenerated by libomptarget-nvptx configuration.\n# Do not edit!")
+configure_file(lit.site.cfg.in lit.site.cfg @ONLY)

libomptarget/deviceRTLs/nvptx/test/api/ignored.c

@@ -0,0 +1,38 @@
+// RUN: %compile-run-and-check
+
+#include <omp.h>
+#include <stdio.h>
+
+const int MaxThreads = 1024;
+
+int main(int argc, char *argv[]) {
+  int cancellation = -1, dynamic = -1, nested = -1, maxActiveLevels = -1;
+
+  #pragma omp target map(cancellation, dynamic, nested, maxActiveLevels)
+  {
+    // libomptarget-nvptx doesn't support cancellation.
+    cancellation = omp_get_cancellation();
+
+    // No support for dynamic adjustment of the number of threads.
+    omp_set_dynamic(1);
+    dynamic = omp_get_dynamic();
+
+    // libomptarget-nvptx doesn't support nested parallelism.
+    omp_set_nested(1);
+    nested = omp_get_nested();
+
+    omp_set_max_active_levels(42);
+    maxActiveLevels = omp_get_max_active_levels();
+  }
+
+  // CHECK: cancellation = 0
+  printf("cancellation = %d\n", cancellation);
+  // CHECK: dynamic = 0
+  printf("dynamic = %d\n", dynamic);
+  // CHECK: nested = 0
+  printf("nested = %d\n", nested);
+  // CHECK: maxActiveLevels = 1
+  printf("maxActiveLevels = %d\n", maxActiveLevels);
+
+  return 0;
+}

libomptarget/deviceRTLs/nvptx/test/data_sharing/alignment.c

@@ -0,0 +1,55 @@
+// RUN: %compile-run-and-check
+
+#include <omp.h>
+#include <stdio.h>
+
+#pragma omp declare target
+static void putValueInParallel(int *ptr, int value) {
+  #pragma omp parallel
+  {
+    *ptr = value;
+  }
+}
+
+static int getId() {
+  int id;
+  putValueInParallel(&id, omp_get_thread_num());
+  return id;
+}
+#pragma omp end declare target
+
+const int MaxThreads = 1024;
+const int Threads = 64;
+
+int main(int argc, char *argv[]) {
+  int master;
+  int check[MaxThreads];
+  for (int i = 0; i < MaxThreads; i++) {
+    check[i] = 0;
+  }
+
+  #pragma omp target map(master, check[:])
+  {
+    master = getId();
+
+    #pragma omp parallel num_threads(Threads)
+    {
+      check[omp_get_thread_num()] = getId();
+    }
+  }
+
+  // CHECK: master = 0.
+  printf("master = %d.\n", master);
+  // CHECK-NOT: invalid
+  for (int i = 0; i < MaxThreads; i++) {
+    if (i < Threads) {
+      if (check[i] != i) {
+        printf("invalid: check[%d] should be %d, is %d\n", i, i, check[i]);
+      }
+    } else if (check[i] != 0) {
+      printf("invalid: check[%d] should be 0, is %d\n", i, check[i]);
+    }
+  }
+
+  return 0;
+}

libomptarget/deviceRTLs/nvptx/test/lit.cfg

@@ -0,0 +1,69 @@
+# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
+# Configuration file for the 'lit' test runner.
+
+import os
+import lit.formats
+
+# Tell pylint that we know config and lit_config exist somewhere.
+if 'PYLINT_IMPORT' in os.environ:
+    config = object()
+    lit_config = object()
+
+def prepend_library_path(name, value, sep):
+    if name in config.environment:
+        config.environment[name] = value + sep + config.environment[name]
+    else:
+        config.environment[name] = value
+
+# name: The name of this test suite.
+config.name = 'libomptarget-nvptx'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.c', '.cpp', '.cc']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+# test_exec_root: The root object directory where output is placed
+config.test_exec_root = config.binary_dir
+
+# test format
+config.test_format = lit.formats.ShTest()
+
+# compiler flags
+config.test_flags = " -I " + config.omp_header_directory + \
+    " -L " + config.library_dir + \
+    " --libomptarget-nvptx-path=" + config.library_dir;
+
+if config.omp_host_rtl_directory:
+    config.test_flags = config.test_flags + \
+        " -L " + config.omp_host_rtl_directory
+
+config.test_flags = config.test_flags + " " + config.test_extra_flags
+
+# Setup environment to find dynamic library at runtime.
+prepend_library_path('LD_LIBRARY_PATH', config.library_dir, ":")
+prepend_library_path('LD_LIBRARY_PATH', config.omp_host_rtl_directory, ":")
+
+# Forbid fallback to host.
+config.environment["OMP_TARGET_OFFLOAD"] = "MANDATORY"
+
+# substitutions
+config.substitutions.append(("%compilexx-run-and-check",
+    "%compilexx-and-run | " + config.libomptarget_filecheck + " %s"))
+config.substitutions.append(("%compile-run-and-check",
+    "%compile-and-run | " + config.libomptarget_filecheck + " %s"))
+config.substitutions.append(("%compilexx-and-run", "%compilexx && %run"))
+config.substitutions.append(("%compile-and-run", "%compile && %run"))
+
+config.substitutions.append(("%compilexx",
+    "%clangxx %openmp_flags %flags %s -o %t"))
+config.substitutions.append(("%compile",
+    "%clang %openmp_flags %flags %s -o %t"))
+
+config.substitutions.append(("%clangxx", config.test_cxx_compiler))
+config.substitutions.append(("%clang", config.test_c_compiler))
+config.substitutions.append(("%openmp_flags", config.test_openmp_flags))
+config.substitutions.append(("%flags", config.test_flags))
+
+config.substitutions.append(("%run", "%t"))

libomptarget/deviceRTLs/nvptx/test/lit.site.cfg.in

@@ -0,0 +1,14 @@
+@AUTO_GEN_COMMENT@
+
+config.test_c_compiler = "@OPENMP_TEST_C_COMPILER@"
+config.test_cxx_compiler = "@OPENMP_TEST_CXX_COMPILER@"
+config.test_openmp_flags = "@LIBOMPTARGET_NVPTX_TEST_OPENMP_FLAGS@"
+config.test_extra_flags = "@LIBOMPTARGET_NVPTX_TEST_FLAGS@"
+config.binary_dir = "@CMAKE_CURRENT_BINARY_DIR@"
+config.library_dir = "@LIBOMPTARGET_LIBRARY_DIR@"
+config.omp_header_directory = "@LIBOMPTARGET_OPENMP_HEADER_FOLDER@"
+config.omp_host_rtl_directory = "@LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER@"
+config.libomptarget_filecheck = "@OPENMP_FILECHECK_EXECUTABLE@"
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")

libomptarget/deviceRTLs/nvptx/test/parallel/level.c

@@ -0,0 +1,139 @@
+// RUN: %compile-run-and-check
+
+#include <omp.h>
+#include <stdio.h>
+
+const int MaxThreads = 1024;
+const int NumThreads = 64;
+
+int main(int argc, char *argv[]) {
+  int level = -1, activeLevel = -1;
+  // The expected value is -1, initialize to different value.
+  int ancestorTNumNeg = 1, teamSizeNeg = 1;
+  int ancestorTNum0 = -1, teamSize0 = -1;
+  // The expected value is -1, initialize to different value.
+  int ancestorTNum1 = 1, teamSize1 = 1;
+  int check1[MaxThreads];
+  int check2[MaxThreads];
+  int check3[MaxThreads];
+  int check4[MaxThreads];
+  for (int i = 0; i < MaxThreads; i++) {
+    check1[i] = check2[i] = check3[i] = check4[i] = 0;
+  }
+
+  #pragma omp target map(level, activeLevel, ancestorTNumNeg, teamSizeNeg) \
+                     map(ancestorTNum0, teamSize0, ancestorTNum1, teamSize1) \
+                     map(check1[:], check2[:], check3[:], check4[:])
+  {
+    level = omp_get_level();
+    activeLevel = omp_get_active_level();
+
+    // Expected to return -1.
+    ancestorTNumNeg = omp_get_ancestor_thread_num(-1);
+    teamSizeNeg = omp_get_team_size(-1);
+
+    // Expected to return 0 and 1.
+    ancestorTNum0 = omp_get_ancestor_thread_num(0);
+    teamSize0 = omp_get_team_size(0);
+
+    // Expected to return -1 because the requested level is larger than
+    // the nest level.
+    ancestorTNum1 = omp_get_ancestor_thread_num(1);
+    teamSize1 = omp_get_team_size(1);
+
+    // Expecting active parallel region.
+    #pragma omp parallel num_threads(NumThreads)
+    {
+      int id = omp_get_thread_num();
+      // Multiply return value of omp_get_level by 5 to avoid that this test
+      // passes if both API calls return wrong values.
+      check1[id] += omp_get_level() * 5 + omp_get_active_level();
+
+      // Expected to return 0 and 1.
+      check2[id] += omp_get_ancestor_thread_num(0) + 5 * omp_get_team_size(0);
+      // Expected to return the current thread num.
+      check2[id] += (omp_get_ancestor_thread_num(1) - id);
+      // Exepcted to return the current number of threads.
+      check2[id] += 3 * omp_get_team_size(1);
+      // Expected to return -1, see above.
+      check2[id] += omp_get_ancestor_thread_num(2) + omp_get_team_size(2);
+
+      // Expecting serialized parallel region.
+      #pragma omp parallel
+      {
+        #pragma omp atomic
+        check3[id] += omp_get_level() * 5 + omp_get_active_level();
+
+        // Expected to return 0 and 1.
+        int check4Inc = omp_get_ancestor_thread_num(0) + 5 * omp_get_team_size(0);
+        // Expected to return the parent thread num.
+        check4Inc += (omp_get_ancestor_thread_num(1) - id);
+        // Exepcted to return the number of threads in the active parallel region.
+        check4Inc += 3 * omp_get_team_size(1);
+        // Exptected to return 0 and 1.
+        check4Inc += omp_get_ancestor_thread_num(2) + 3 * omp_get_team_size(2);
+        // Expected to return -1, see above.
+        check4Inc += omp_get_ancestor_thread_num(3) + omp_get_team_size(3);
+
+        #pragma omp atomic
+        check4[id] += check4Inc;
+      }
+    }
+  }
+
+  // CHECK: target: level = 0, activeLevel = 0
+  printf("target: level = %d, activeLevel = %d\n", level, activeLevel);
+  // CHECK: level = -1: ancestorTNum = -1, teamSize = -1
+  printf("level = -1: ancestorTNum = %d, teamSize = %d\n", ancestorTNumNeg, teamSizeNeg);
+  // CHECK: level = 0: ancestorTNum = 0, teamSize = 1
+  printf("level = 0: ancestorTNum = %d, teamSize = %d\n", ancestorTNum0, teamSize0);
+  // CHECK: level = 1: ancestorTNum = -1, teamSize = -1
+  printf("level = 1: ancestorTNum = %d, teamSize = %d\n", ancestorTNum1, teamSize1);
+
+  // CHECK-NOT: invalid
+  for (int i = 0; i < MaxThreads; i++) {
+    // Check active parallel region:
+    // omp_get_level() = 1, omp_get_active_level() = 1
+    const int Expected1 = 6;
+    if (i < NumThreads) {
+      if (check1[i] != Expected1) {
+        printf("invalid: check1[%d] should be %d, is %d\n", i, Expected1, check1[i]);
+      }
+    } else if (check1[i] != 0) {
+      printf("invalid: check1[%d] should be 0, is %d\n", i, check1[i]);
+    }
+
+    // 5 * 1 + 3 * 64 - 1 - 1 (see above)
+    const int Expected2 = 195;
+    if (i < NumThreads) {
+      if (check2[i] != Expected2) {
+        printf("invalid: check2[%d] should be %d, is %d\n", i, Expected2, check2[i]);
+      }
+    } else if (check2[i] != 0) {
+      printf("invalid: check2[%d] should be 0, is %d\n", i, check2[i]);
+    }
+
+    // Check serialized parallel region:
+    // omp_get_level() = 2, omp_get_active_level() = 1
+    const int Expected3 = 11;
+    if (i < NumThreads) {
+      if (check3[i] != Expected3) {
+        printf("invalid: check3[%d] should be %d, is %d\n", i, Expected3, check3[i]);
+      }
+    } else if (check3[i] != 0) {
+      printf("invalid: check3[%d] should be 0, is %d\n", i, check3[i]);
+    }
+
+    // 5 * 1 + 3 * 64 + 3 * 1 - 1 - 1 (see above)
+    const int Expected4 = 198;
+    if (i < NumThreads) {
+      if (check4[i] != Expected4) {
+        printf("invalid: check4[%d] should be %d, is %d\n", i, Expected4, check4[i]);
+      }
+    } else if (check4[i] != 0) {
+      printf("invalid: check4[%d] should be 0, is %d\n", i, check4[i]);
+    }
+  }
+
+  return 0;
+}

libomptarget/deviceRTLs/nvptx/test/parallel/nested.c

@@ -0,0 +1,72 @@
+// RUN: %compile-run-and-check
+
+#include <omp.h>
+#include <stdio.h>
+
+const int MaxThreads = 1024;
+const int NumThreads = 64;
+
+int main(int argc, char *argv[]) {
+  int inParallel = -1, numThreads = -1, threadNum = -1;
+  int check1[MaxThreads];
+  int check2[MaxThreads];
+  for (int i = 0; i < MaxThreads; i++) {
+    check1[i] = check2[i] = 0;
+  }
+
+  #pragma omp target map(inParallel, numThreads, threadNum, check1[:], check2[:])
+  {
+    inParallel = omp_in_parallel();
+    numThreads = omp_get_num_threads();
+    threadNum = omp_get_thread_num();
+
+    // Expecting active parallel region.
+    #pragma omp parallel num_threads(NumThreads)
+    {
+      int id = omp_get_thread_num();
+      check1[id] += omp_get_num_threads() + omp_in_parallel();
+
+      // Expecting serialized parallel region.
+      #pragma omp parallel
+      {
+        // Expected to be 1.
+        int nestedInParallel = omp_in_parallel();
+        // Expected to be 1.
+        int nestedNumThreads = omp_get_num_threads();
+        // Expected to be 0.
+        int nestedThreadNum = omp_get_thread_num();
+        #pragma omp atomic
+        check2[id] += nestedInParallel + nestedNumThreads + nestedThreadNum;
+      }
+    }
+  }
+
+  // CHECK: target: inParallel = 0, numThreads = 1, threadNum = 0
+  printf("target: inParallel = %d, numThreads = %d, threadNum = %d\n",
+         inParallel, numThreads, threadNum);
+
+  // CHECK-NOT: invalid
+  for (int i = 0; i < MaxThreads; i++) {
+    // Check that all threads reported
+    // omp_get_num_threads() = 64, omp_in_parallel() = 1.
+    int Expected = NumThreads + 1;
+    if (i < NumThreads) {
+      if (check1[i] != Expected) {
+        printf("invalid: check1[%d] should be %d, is %d\n", i, Expected, check1[i]);
+      }
+    } else if (check1[i] != 0) {
+      printf("invalid: check1[%d] should be 0, is %d\n", i, check1[i]);
+    }
+
+    // Check serialized parallel region.
+    if (i < NumThreads) {
+      if (check2[i] != 2) {
+        printf("invalid: check2[%d] should be 2, is %d\n", i, check2[i]);
+      }
+    } else if (check2[i] != 0) {
+      printf("invalid: check2[%d] should be 0, is %d\n", i, check2[i]);
+    }
+  }
+
+  return 0;
+}

libomptarget/deviceRTLs/nvptx/test/parallel/num_threads.c

@@ -0,0 +1,102 @@
+// RUN: %compile-run-and-check
+
+#include <stdio.h>
+#include <omp.h>
+
+const int WarpSize = 32;
+const int NumThreads1 = 1 * WarpSize;
+const int NumThreads2 = 2 * WarpSize;
+const int NumThreads3 = 3 * WarpSize;
+const int MaxThreads = 1024;
+
+int main(int argc, char *argv[]) {
+  int check1[MaxThreads];
+  int check2[MaxThreads];
+  int check3[MaxThreads];
+  int check4[MaxThreads];
+  for (int i = 0; i < MaxThreads; i++) {
+    check1[i] = check2[i] = check3[i] = check4[i] = 0;
+  }
+
+  int maxThreads1 = -1;
+  int maxThreads2 = -1;
+  int maxThreads3 = -1;
+
+  #pragma omp target map(check1[:], check2[:], check3[:], check4[:]) \
+                     map(maxThreads1, maxThreads2, maxThreads3)
+  {
+    #pragma omp parallel num_threads(NumThreads1)
+    {
+      check1[omp_get_thread_num()] += omp_get_num_threads();
+    }
+
+    // API method to set number of threads in parallel regions without
+    // num_threads() clause.
+    omp_set_num_threads(NumThreads2);
+    maxThreads1 = omp_get_max_threads();
+    #pragma omp parallel
+    {
+      check2[omp_get_thread_num()] += omp_get_num_threads();
+    }
+
+    maxThreads2 = omp_get_max_threads();
+
+    // num_threads() clause should override nthreads-var ICV.
+    #pragma omp parallel num_threads(NumThreads3)
+    {
+      check3[omp_get_thread_num()] += omp_get_num_threads();
+    }
+
+    maxThreads3 = omp_get_max_threads();
+
+    // Effect from omp_set_num_threads() should still be visible.
+    #pragma omp parallel
+    {
+      check4[omp_get_thread_num()] += omp_get_num_threads();
+    }
+  }
+
+  // CHECK: maxThreads1 = 64
+  printf("maxThreads1 = %d\n", maxThreads1);
+  // CHECK: maxThreads2 = 64
+  printf("maxThreads2 = %d\n", maxThreads2);
+  // CHECK: maxThreads3 = 64
+  printf("maxThreads3 = %d\n", maxThreads3);
+
+  // CHECK-NOT: invalid
+  for (int i = 0; i < MaxThreads; i++) {
+    if (i < NumThreads1) {
+      if (check1[i] != NumThreads1) {
+        printf("invalid: check1[%d] should be %d, is %d\n", i, NumThreads1, check1[i]);
+      }
+    } else if (check1[i] != 0) {
+      printf("invalid: check1[%d] should be 0, is %d\n", i, check1[i]);
+    }
+
+    if (i < NumThreads2) {
+      if (check2[i] != NumThreads2) {
+        printf("invalid: check2[%d] should be %d, is %d\n", i, NumThreads2, check2[i]);
+      }
+    } else if (check2[i] != 0) {
+      printf("invalid: check2[%d] should be 0, is %d\n", i, check2[i]);
+    }
+
+    if (i < NumThreads3) {
+      if (check3[i] != NumThreads3) {
+        printf("invalid: check3[%d] should be %d, is %d\n", i, NumThreads3, check3[i]);
+      }
+    } else if (check3[i] != 0) {
+      printf("invalid: check3[%d] should be 0, is %d\n", i, check3[i]);
+    }
+
+    if (i < NumThreads2) {
+      if (check4[i] != NumThreads2) {
+        printf("invalid: check4[%d] should be %d, is %d\n", i, NumThreads2, check4[i]);
+      }
+    } else if (check4[i] != 0) {
+      printf("invalid: check4[%d] should be 0, is %d\n", i, check4[i]);
+    }
+  }
+
+  return 0;
+}

libomptarget/deviceRTLs/nvptx/test/parallel/thread_limit.c

@@ -0,0 +1,77 @@
+// RUN: %compile-run-and-check
+
+#include <stdio.h>
+#include <omp.h>
+
+const int WarpSize = 32;
+const int ThreadLimit = 1 * WarpSize;
+const int NumThreads2 = 2 * WarpSize;
+const int NumThreads3 = 3 * WarpSize;
+const int MaxThreads = 1024;
+
+int main(int argc, char *argv[]) {
+  int check1[MaxThreads];
+  int check2[MaxThreads];
+  int check3[MaxThreads];
+  for (int i = 0; i < MaxThreads; i++) {
+    check1[i] = check2[i] = check3[i] = 0;
+  }
+
+  int threadLimit = -1;
+
+  #pragma omp target teams num_teams(1) thread_limit(ThreadLimit) \
+                           map(check1[:], check2[:], check3[:], threadLimit)
+  {
+    threadLimit = omp_get_thread_limit();
+
+    // All parallel regions should get as many threads as specified by the
+    // thread_limit() clause.
+    #pragma omp parallel
+    {
+      check1[omp_get_thread_num()] += omp_get_num_threads();
+    }
+
+    omp_set_num_threads(NumThreads2);
+    #pragma omp parallel
+    {
+      check2[omp_get_thread_num()] += omp_get_num_threads();
+    }
+
+    #pragma omp parallel num_threads(NumThreads3)
+    {
+      check3[omp_get_thread_num()] += omp_get_num_threads();
+    }
+  }
+
+  // CHECK: threadLimit = 32
+  printf("threadLimit = %d\n", threadLimit);
+
+  // CHECK-NOT: invalid
+  for (int i = 0; i < MaxThreads; i++) {
+    if (i < ThreadLimit) {
+      if (check1[i] != ThreadLimit) {
+        printf("invalid: check1[%d] should be %d, is %d\n", i, ThreadLimit, check1[i]);
+      }
+    } else if (check1[i] != 0) {
+      printf("invalid: check1[%d] should be 0, is %d\n", i, check1[i]);
+    }
+
+    if (i < ThreadLimit) {
+      if (check2[i] != ThreadLimit) {
+        printf("invalid: check2[%d] should be %d, is %d\n", i, ThreadLimit, check2[i]);
+      }
+    } else if (check2[i] != 0) {
+      printf("invalid: check2[%d] should be 0, is %d\n", i, check2[i]);
+    }
+
+    if (i < ThreadLimit) {
+      if (check3[i] != ThreadLimit) {
+        printf("invalid: check3[%d] should be %d, is %d\n", i, ThreadLimit, check3[i]);
+      }
+    } else if (check3[i] != 0) {
+      printf("invalid: check3[%d] should be 0, is %d\n", i, check3[i]);
+    }
+  }
+
+  return 0;
+}

libomptarget/include/omptarget.h

@@ -48,7 +48,7 @@ enum tgt_map_type {
   OMP_TGT_MAPTYPE_LITERAL         = 0x100,
   // mapping is implicit
   OMP_TGT_MAPTYPE_IMPLICIT        = 0x200,
-  // member of struct, member given by 16 MSBs - 1
+  // member of struct, member given by [16 MSBs] - 1
   OMP_TGT_MAPTYPE_MEMBER_OF       = 0xffff000000000000
 };
 
@@ -99,10 +99,6 @@ struct __tgt_target_table {
 extern "C" {
 #endif
 
-// Implemented in libomp, they are called from within __tgt_* functions.
-int omp_get_default_device(void) __attribute__((weak));
-int32_t __kmpc_omp_taskwait(void *loc_ref, int32_t gtid) __attribute__((weak));
-
 int omp_get_num_devices(void);
 int omp_get_initial_device(void);
 void *omp_target_alloc(size_t size, int device_num);

libomptarget/plugins/CMakeLists.txt

@@ -37,7 +37,7 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$")
         
       # Install plugin under the lib destination folder.
       install(TARGETS "omptarget.rtl.${tmachine_libname}" 
-        LIBRARY DESTINATION lib${OPENMP_LIBDIR_SUFFIX})
+        LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
         
       target_link_libraries(
         "omptarget.rtl.${tmachine_libname}"

libomptarget/plugins/cuda/CMakeLists.txt

@@ -10,41 +10,41 @@
 # Build a plugin for a CUDA machine if available.
 #
 ##===----------------------------------------------------------------------===##
-if(LIBOMPTARGET_DEP_LIBELF_FOUND)
-  if(LIBOMPTARGET_DEP_CUDA_FOUND)
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux")
-    
-      libomptarget_say("Building CUDA offloading plugin.")
-    
-      # Define the suffix for the runtime messaging dumps.
-      add_definitions(-DTARGET_NAME=CUDA)
-    
-      if(LIBOMPTARGET_CMAKE_BUILD_TYPE MATCHES debug)
-          add_definitions(-DCUDA_ERROR_REPORT)
-      endif()
-      
-      include_directories(${LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS})
-      include_directories(${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIRS})
-    
-      add_library(omptarget.rtl.cuda SHARED src/rtl.cpp)
-      
-      # Install plugin under the lib destination folder.
-      install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION lib${OPENMP_LIBDIR_SUFFIX})
-      
-      target_link_libraries(omptarget.rtl.cuda 
-        ${LIBOMPTARGET_DEP_CUDA_LIBRARIES} 
-        cuda 
-        ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
-        "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
-    
-      # Report to the parent scope that we are building a plugin for CUDA.
-      set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda" PARENT_SCOPE)
-    else()
-      libomptarget_say("Not building CUDA offloading plugin: only support CUDA in Linux x86_64 or ppc64le hosts.")
-    endif()
-  else()
-    libomptarget_say("Not building CUDA offloading plugin: CUDA not found in system.")
-  endif()
-else(LIBOMPTARGET_DEP_LIBELF_FOUND)
+if (NOT(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(ppc64le)$" AND CMAKE_SYSTEM_NAME MATCHES "Linux"))
+  libomptarget_say("Not building CUDA offloading plugin: only support CUDA in Linux x86_64 or ppc64le hosts.")
+  return()
+elseif (NOT LIBOMPTARGET_DEP_LIBELF_FOUND)
   libomptarget_say("Not building CUDA offloading plugin: libelf dependency not found.")
-endif(LIBOMPTARGET_DEP_LIBELF_FOUND)
+  return()
+elseif(NOT LIBOMPTARGET_DEP_CUDA_FOUND)
+  libomptarget_say("Not building CUDA offloading plugin: CUDA not found in system.")
+  return()
+elseif(NOT LIBOMPTARGET_DEP_CUDA_DRIVER_FOUND)
+  libomptarget_say("Not building CUDA offloading plugin: CUDA Driver API not found in system.")
+  return()
+endif()
+
+libomptarget_say("Building CUDA offloading plugin.")
+
+# Define the suffix for the runtime messaging dumps.
+add_definitions(-DTARGET_NAME=CUDA)
+
+if(LIBOMPTARGET_CMAKE_BUILD_TYPE MATCHES debug)
+    add_definitions(-DCUDA_ERROR_REPORT)
+endif()
+
+include_directories(${LIBOMPTARGET_DEP_CUDA_INCLUDE_DIRS})
+include_directories(${LIBOMPTARGET_DEP_LIBELF_INCLUDE_DIRS})
+
+add_library(omptarget.rtl.cuda SHARED src/rtl.cpp)
+
+# Install plugin under the lib destination folder.
+install(TARGETS omptarget.rtl.cuda LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}")
+
+target_link_libraries(omptarget.rtl.cuda
+  ${LIBOMPTARGET_DEP_CUDA_DRIVER_LIBRARIES}
+  ${LIBOMPTARGET_DEP_LIBELF_LIBRARIES}
+  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports")
+
+# Report to the parent scope that we are building a plugin for CUDA.
+set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS} nvptx64-nvidia-cuda" PARENT_SCOPE)

libomptarget/plugins/cuda/src/rtl.cpp

@@ -14,7 +14,6 @@
 #include <cassert>
 #include <cstddef>
 #include <cuda.h>
-#include <cuda_runtime_api.h>
 #include <list>
 #include <string>
 #include <vector>
@@ -81,13 +80,19 @@ struct KernelTy {
       : Func(_Func), ExecutionMode(_ExecutionMode) {}
 };
 
+/// Device envrionment data
+/// Manually sync with the deviceRTL side for now, move to a dedicated header file later.
+struct omptarget_device_environmentTy {
+  int32_t debug_level;
+};
+
 /// List that contains all the kernels.
 /// FIXME: we may need this to be per device and per library.
 std::list<KernelTy> KernelsList;
 
 /// Class containing all the device information.
 class RTLDeviceInfoTy {
-  std::vector<FuncOrGblEntryTy> FuncGblEntries;
+  std::vector<std::list<FuncOrGblEntryTy>> FuncGblEntries;
 
 public:
   int NumberOfDevices;
@@ -117,7 +122,7 @@ public:
   void addOffloadEntry(int32_t device_id, __tgt_offload_entry entry) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     E.Entries.push_back(entry);
   }
@@ -126,7 +131,7 @@ public:
   bool findOffloadEntry(int32_t device_id, void *addr) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     for (auto &it : E.Entries) {
       if (it.addr == addr)
@@ -140,7 +145,7 @@ public:
   __tgt_target_table *getOffloadEntriesTable(int32_t device_id) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     int32_t size = E.Entries.size();
 
@@ -162,7 +167,8 @@ public:
   void clearOffloadEntriesTable(int32_t device_id) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncGblEntries[device_id].emplace_back();
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
     E.Entries.clear();
     E.Table.EntriesBegin = E.Table.EntriesEnd = 0;
   }
@@ -279,43 +285,48 @@ int32_t __tgt_rtl_init_device(int32_t device_id) {
     return OFFLOAD_FAIL;
   }
 
-  // scan properties to determine number of threads/block and blocks/grid.
-  struct cudaDeviceProp Properties;
-  cudaError_t error = cudaGetDeviceProperties(&Properties, device_id);
-  if (error != cudaSuccess) {
-    DP("Error getting device Properties, use defaults\n");
+  // Query attributes to determine number of threads/block and blocks/grid.
+  int maxGridDimX;
+  err = cuDeviceGetAttribute(&maxGridDimX, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X,
+                             cuDevice);
+  if (err != CUDA_SUCCESS) {
+    DP("Error getting max grid dimension, use default\n");
     DeviceInfo.BlocksPerGrid[device_id] = RTLDeviceInfoTy::DefaultNumTeams;
+  } else if (maxGridDimX <= RTLDeviceInfoTy::HardTeamLimit) {
+    DeviceInfo.BlocksPerGrid[device_id] = maxGridDimX;
+    DP("Using %d CUDA blocks per grid\n", maxGridDimX);
+  } else {
+    DeviceInfo.BlocksPerGrid[device_id] = RTLDeviceInfoTy::HardTeamLimit;
+    DP("Max CUDA blocks per grid %d exceeds the hard team limit %d, capping "
+       "at the hard limit\n",
+       maxGridDimX, RTLDeviceInfoTy::HardTeamLimit);
+  }
+
+  // We are only exploiting threads along the x axis.
+  int maxBlockDimX;
+  err = cuDeviceGetAttribute(&maxBlockDimX, CU_DEVICE_ATTRIBUTE_MAX_BLOCK_DIM_X,
+                             cuDevice);
+  if (err != CUDA_SUCCESS) {
+    DP("Error getting max block dimension, use default\n");
     DeviceInfo.ThreadsPerBlock[device_id] = RTLDeviceInfoTy::DefaultNumThreads;
+  } else if (maxBlockDimX <= RTLDeviceInfoTy::HardThreadLimit) {
+    DeviceInfo.ThreadsPerBlock[device_id] = maxBlockDimX;
+    DP("Using %d CUDA threads per block\n", maxBlockDimX);
+  } else {
+    DeviceInfo.ThreadsPerBlock[device_id] = RTLDeviceInfoTy::HardThreadLimit;
+    DP("Max CUDA threads per block %d exceeds the hard thread limit %d, capping"
+       "at the hard limit\n",
+       maxBlockDimX, RTLDeviceInfoTy::HardThreadLimit);
+  }
+
+  int warpSize;
+  err =
+      cuDeviceGetAttribute(&warpSize, CU_DEVICE_ATTRIBUTE_WARP_SIZE, cuDevice);
+  if (err != CUDA_SUCCESS) {
+    DP("Error getting warp size, assume default\n");
     DeviceInfo.WarpSize[device_id] = 32;
   } else {
-    // Get blocks per grid
-    if (Properties.maxGridSize[0] <= RTLDeviceInfoTy::HardTeamLimit) {
-      DeviceInfo.BlocksPerGrid[device_id] = Properties.maxGridSize[0];
-      DP("Using %d CUDA blocks per grid\n", Properties.maxGridSize[0]);
-    } else {
-      DeviceInfo.BlocksPerGrid[device_id] = RTLDeviceInfoTy::HardTeamLimit;
-      DP("Max CUDA blocks per grid %d exceeds the hard team limit %d, capping "
-          "at the hard limit\n", Properties.maxGridSize[0],
-          RTLDeviceInfoTy::HardTeamLimit);
-    }
-
-    // Get threads per block, exploit threads only along x axis
-    if (Properties.maxThreadsDim[0] <= RTLDeviceInfoTy::HardThreadLimit) {
-      DeviceInfo.ThreadsPerBlock[device_id] = Properties.maxThreadsDim[0];
-      DP("Using %d CUDA threads per block\n", Properties.maxThreadsDim[0]);
-      if (Properties.maxThreadsDim[0] < Properties.maxThreadsPerBlock) {
-        DP("(fewer than max per block along all xyz dims %d)\n",
-            Properties.maxThreadsPerBlock);
-      }
-    } else {
-      DeviceInfo.ThreadsPerBlock[device_id] = RTLDeviceInfoTy::HardThreadLimit;
-      DP("Max CUDA threads per block %d exceeds the hard thread limit %d, "
-          "capping at the hard limit\n", Properties.maxThreadsDim[0],
-          RTLDeviceInfoTy::HardThreadLimit);
-    }
-
-    // Get warp size
-    DeviceInfo.WarpSize[device_id] = Properties.warpSize;
+    DeviceInfo.WarpSize[device_id] = warpSize;
   }
 
   // Adjust teams to the env variables
@@ -487,6 +498,48 @@ __tgt_target_table *__tgt_rtl_load_binary(int32_t device_id,
     DeviceInfo.addOffloadEntry(device_id, entry);
   }
 
+  // send device environment data to the device
+  {
+    omptarget_device_environmentTy device_env;
+
+    device_env.debug_level = 0;
+
+#ifdef OMPTARGET_DEBUG
+    if (char *envStr = getenv("LIBOMPTARGET_DEVICE_RTL_DEBUG")) {
+      device_env.debug_level = std::stoi(envStr);
+    }
+#endif
+
+    const char * device_env_Name="omptarget_device_environment";
+    CUdeviceptr device_env_Ptr;
+    size_t cusize;
+
+    err = cuModuleGetGlobal(&device_env_Ptr, &cusize, cumod, device_env_Name);
+
+    if (err == CUDA_SUCCESS) {
+      if ((size_t)cusize != sizeof(device_env)) {
+        DP("Global device_environment '%s' - size mismatch (%zu != %zu)\n",
+            device_env_Name, cusize, sizeof(int32_t));
+        CUDA_ERR_STRING(err);
+        return NULL;
+      }
+
+      err = cuMemcpyHtoD(device_env_Ptr, &device_env, cusize);
+      if (err != CUDA_SUCCESS) {
+        DP("Error when copying data from host to device. Pointers: "
+            "host = " DPxMOD ", device = " DPxMOD ", size = %zu\n",
+            DPxPTR(&device_env), DPxPTR(device_env_Ptr), cusize);
+        CUDA_ERR_STRING(err);
+        return NULL;
+      }
+
+      DP("Sending global device environment data %zu bytes\n", (size_t)cusize);
+    } else {
+      DP("Finding global device environment '%s' - symbol missing.\n", device_env_Name);
+      DP("Continue, considering this is a device RTL which does not accept envrionment setting.\n");
+    }
+  }
+
   return DeviceInfo.getOffloadEntriesTable(device_id);
 }
 
@@ -678,17 +731,16 @@ int32_t __tgt_rtl_run_target_team_region(int32_t device_id, void *tgt_entry_ptr,
   if (err != CUDA_SUCCESS) {
     DP("Device kernel launch failed!\n");
     CUDA_ERR_STRING(err);
-    assert(err == CUDA_SUCCESS && "Unable to launch target execution!");
     return OFFLOAD_FAIL;
   }
 
   DP("Launch of entry point at " DPxMOD " successful!\n",
       DPxPTR(tgt_entry_ptr));
 
-  cudaError_t sync_error = cudaDeviceSynchronize();
-  if (sync_error != cudaSuccess) {
-  DP("Kernel execution error at " DPxMOD ", %s.\n", DPxPTR(tgt_entry_ptr),
-      cudaGetErrorString(sync_error));
+  CUresult sync_err = cuCtxSynchronize();
+  if (sync_err != CUDA_SUCCESS) {
+    DP("Kernel execution error at " DPxMOD "!\n", DPxPTR(tgt_entry_ptr));
+    CUDA_ERR_STRING(sync_err);
     return OFFLOAD_FAIL;
   } else {
     DP("Kernel execution at " DPxMOD " successful!\n", DPxPTR(tgt_entry_ptr));

libomptarget/plugins/generic-elf-64bit/src/rtl.cpp

@@ -66,7 +66,7 @@ struct FuncOrGblEntryTy {
 
 /// Class containing all the device information.
 class RTLDeviceInfoTy {
-  std::vector<FuncOrGblEntryTy> FuncGblEntries;
+  std::vector<std::list<FuncOrGblEntryTy>> FuncGblEntries;
 
 public:
   std::list<DynLibTy> DynLibs;
@@ -76,7 +76,8 @@ public:
                           __tgt_offload_entry *end) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncGblEntries[device_id].emplace_back();
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     E.Table.EntriesBegin = begin;
     E.Table.EntriesEnd = end;
@@ -86,7 +87,7 @@ public:
   bool findOffloadEntry(int32_t device_id, void *addr) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     for (__tgt_offload_entry *i = E.Table.EntriesBegin, *e = E.Table.EntriesEnd;
          i < e; ++i) {
@@ -101,7 +102,7 @@ public:
   __tgt_target_table *getOffloadEntriesTable(int32_t device_id) {
     assert(device_id < (int32_t)FuncGblEntries.size() &&
            "Unexpected device id!");
-    FuncOrGblEntryTy &E = FuncGblEntries[device_id];
+    FuncOrGblEntryTy &E = FuncGblEntries[device_id].back();
 
     return &E.Table;
   }

libomptarget/src/CMakeLists.txt

@@ -28,4 +28,5 @@ target_link_libraries(omptarget
   "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
 
 # Install libomptarget under the lib destination folder.
-install(TARGETS omptarget LIBRARY DESTINATION lib${OPENMP_LIBDIR_SUFFIX})
+install(TARGETS omptarget LIBRARY COMPONENT omptarget
+  DESTINATION "${OPENMP_INSTALL_LIBDIR}")

libomptarget/src/api.cpp

@@ -19,6 +19,7 @@
 
 #include <climits>
 #include <cstring>
+#include <cstdlib>
 
 EXTERN int omp_get_num_devices(void) {
   RTLsMtx.lock();

libomptarget/src/interface.cpp

@@ -19,6 +19,55 @@
 #include "rtl.h"
 
 #include <cassert>
+#include <cstdlib>
+#include <mutex>
+
+// Store target policy (disabled, mandatory, default)
+kmp_target_offload_kind_t TargetOffloadPolicy = tgt_default;
+std::mutex TargetOffloadMtx;
+
+////////////////////////////////////////////////////////////////////////////////
+/// manage the success or failure of a target constuct
+
+static void HandleDefaultTargetOffload() {
+  TargetOffloadMtx.lock();
+  if (TargetOffloadPolicy == tgt_default) {
+    if (omp_get_num_devices() > 0) {
+      DP("Default TARGET OFFLOAD policy is now mandatory "
+         "(devicew were found)\n");
+      TargetOffloadPolicy = tgt_mandatory;
+    } else {
+      DP("Default TARGET OFFLOAD policy is now disabled "
+         "(devices were not found)\n");
+      TargetOffloadPolicy = tgt_disabled;
+    }
+  }
+  TargetOffloadMtx.unlock();
+}
+
+static int IsOffloadDisabled() {
+  if (TargetOffloadPolicy == tgt_default) HandleDefaultTargetOffload();
+  return TargetOffloadPolicy == tgt_disabled;
+}
+
+static void HandleTargetOutcome(bool success) {
+  switch (TargetOffloadPolicy) {
+    case tgt_disabled:
+      if (success) {
+        FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
+      }
+      break;
+    case tgt_default:
+        FATAL_MESSAGE0(1, "default offloading policy must switched to " 
+            "mandatory or disabled");
+      break;
+    case tgt_mandatory:
+      if (!success) {
+        FATAL_MESSAGE0(1, "failure of target construct while offloading is mandatory");
+      }
+      break;
+  }
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 /// adds a target shared library to the target execution image
@@ -32,265 +81,41 @@ EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
   RTLs.UnregisterLib(desc);
 }
 
-// Following datatypes and functions (tgt_oldmap_type, combined_entry_t,
-// translate_map, cleanup_map) will be removed once the compiler starts using
-// the new map types.
-
-// Old map types
-enum tgt_oldmap_type {
-  OMP_TGT_OLDMAPTYPE_TO          = 0x001, // copy data from host to device
-  OMP_TGT_OLDMAPTYPE_FROM        = 0x002, // copy data from device to host
-  OMP_TGT_OLDMAPTYPE_ALWAYS      = 0x004, // copy regardless of the ref. count
-  OMP_TGT_OLDMAPTYPE_DELETE      = 0x008, // force unmapping of data
-  OMP_TGT_OLDMAPTYPE_MAP_PTR     = 0x010, // map pointer as well as pointee
-  OMP_TGT_OLDMAPTYPE_FIRST_MAP   = 0x020, // first occurrence of mapped variable
-  OMP_TGT_OLDMAPTYPE_RETURN_PTR  = 0x040, // return TgtBase addr of mapped data
-  OMP_TGT_OLDMAPTYPE_PRIVATE_PTR = 0x080, // private variable - not mapped
-  OMP_TGT_OLDMAPTYPE_PRIVATE_VAL = 0x100  // copy by value - not mapped
-};
-
-// Temporary functions for map translation and cleanup
-struct combined_entry_t {
-  int num_members; // number of members in combined entry
-  void *base_addr; // base address of combined entry
-  void *begin_addr; // begin address of combined entry
-  void *end_addr; // size of combined entry
-};
-
-static void translate_map(int32_t arg_num, void **args_base, void **args,
-    int64_t *arg_sizes, int64_t *arg_types, int32_t &new_arg_num,
-    void **&new_args_base, void **&new_args, int64_t *&new_arg_sizes,
-    int64_t *&new_arg_types, bool is_target_construct) {
-  if (arg_num <= 0) {
-    DP("Nothing to translate\n");
-    new_arg_num = 0;
-    return;
-  }
-
-  // array of combined entries
-  combined_entry_t *cmb_entries =
-      (combined_entry_t *) alloca(arg_num * sizeof(combined_entry_t));
-  // number of combined entries
-  long num_combined = 0;
-  // old entry is MAP_PTR?
-  bool *is_ptr_old = (bool *) alloca(arg_num * sizeof(bool));
-  // old entry is member of member_of[old] cmb_entry
-  int *member_of = (int *) alloca(arg_num * sizeof(int));
-  // temporary storage for modifications of the original arg_types
-  int64_t *mod_arg_types = (int64_t *) alloca(arg_num  *sizeof(int64_t));
-
-  DP("Translating %d map entries\n", arg_num);
-  for (int i = 0; i < arg_num; ++i) {
-    member_of[i] = -1;
-    is_ptr_old[i] = false;
-    mod_arg_types[i] = arg_types[i];
-    // Scan previous entries to see whether this entry shares the same base
-    for (int j = 0; j < i; ++j) {
-      void *new_begin_addr = NULL;
-      void *new_end_addr = NULL;
-
-      if (mod_arg_types[i] & OMP_TGT_OLDMAPTYPE_MAP_PTR) {
-        if (args_base[i] == args[j]) {
-          if (!(mod_arg_types[j] & OMP_TGT_OLDMAPTYPE_MAP_PTR)) {
-            DP("Entry %d has the same base as entry %d's begin address\n", i,
-                j);
-            new_begin_addr = args_base[i];
-            new_end_addr = (char *)args_base[i] + sizeof(void *);
-            assert(arg_sizes[j] == sizeof(void *));
-            is_ptr_old[j] = true;
-          } else {
-            DP("Entry %d has the same base as entry %d's begin address, but "
-                "%d's base was a MAP_PTR too\n", i, j, j);
-            int32_t to_from_always_delete =
-                OMP_TGT_OLDMAPTYPE_TO | OMP_TGT_OLDMAPTYPE_FROM |
-                OMP_TGT_OLDMAPTYPE_ALWAYS | OMP_TGT_OLDMAPTYPE_DELETE;
-            if (mod_arg_types[j] & to_from_always_delete) {
-              DP("Resetting to/from/always/delete flags for entry %d because "
-                  "it is only a pointer to pointer\n", j);
-              mod_arg_types[j] &= ~to_from_always_delete;
-            }
-          }
-        }
-      } else {
-        if (!(mod_arg_types[i] & OMP_TGT_OLDMAPTYPE_FIRST_MAP) &&
-            args_base[i] == args_base[j]) {
-          DP("Entry %d has the same base address as entry %d\n", i, j);
-          new_begin_addr = args[i];
-          new_end_addr = (char *)args[i] + arg_sizes[i];
-        }
-      }
-
-      // If we have combined the entry with a previous one
-      if (new_begin_addr) {
-        int id;
-        if(member_of[j] == -1) {
-          // We have a new entry
-          id = num_combined++;
-          DP("Creating new combined entry %d for old entry %d\n", id, j);
-          // Initialize new entry
-          cmb_entries[id].num_members = 1;
-          cmb_entries[id].base_addr = args_base[j];
-          if (mod_arg_types[j] & OMP_TGT_OLDMAPTYPE_MAP_PTR) {
-            cmb_entries[id].begin_addr = args_base[j];
-            cmb_entries[id].end_addr = (char *)args_base[j] + arg_sizes[j];
-          } else {
-            cmb_entries[id].begin_addr = args[j];
-            cmb_entries[id].end_addr = (char *)args[j] + arg_sizes[j];
-          }
-          member_of[j] = id;
-        } else {
-          // Reuse existing combined entry
-          DP("Reusing existing combined entry %d\n", member_of[j]);
-          id = member_of[j];
-        }
-
-        // Update combined entry
-        DP("Adding entry %d to combined entry %d\n", i, id);
-        cmb_entries[id].num_members++;
-        // base_addr stays the same
-        cmb_entries[id].begin_addr =
-            std::min(cmb_entries[id].begin_addr, new_begin_addr);
-        cmb_entries[id].end_addr =
-            std::max(cmb_entries[id].end_addr, new_end_addr);
-        member_of[i] = id;
-        break;
-      }
-    }
-  }
-
-  DP("New entries: %ld combined + %d original\n", num_combined, arg_num);
-  new_arg_num = arg_num + num_combined;
-  new_args_base = (void **) malloc(new_arg_num * sizeof(void *));
-  new_args = (void **) malloc(new_arg_num * sizeof(void *));
-  new_arg_sizes = (int64_t *) malloc(new_arg_num * sizeof(int64_t));
-  new_arg_types = (int64_t *) malloc(new_arg_num * sizeof(int64_t));
-
-  const int64_t alignment = 8;
-
-  int next_id = 0; // next ID
-  int next_cid = 0; // next combined ID
-  int *combined_to_new_id = (int *) alloca(num_combined * sizeof(int));
-  for (int i = 0; i < arg_num; ++i) {
-    // It is member_of
-    if (member_of[i] == next_cid) {
-      int cid = next_cid++; // ID of this combined entry
-      int nid = next_id++; // ID of the new (global) entry
-      combined_to_new_id[cid] = nid;
-      DP("Combined entry %3d will become new entry %3d\n", cid, nid);
-
-      int64_t padding = (int64_t)cmb_entries[cid].begin_addr % alignment;
-      if (padding) {
-        DP("Using a padding of %" PRId64 " for begin address " DPxMOD "\n",
-            padding, DPxPTR(cmb_entries[cid].begin_addr));
-        cmb_entries[cid].begin_addr =
-            (char *)cmb_entries[cid].begin_addr - padding;
-      }
-
-      new_args_base[nid] = cmb_entries[cid].base_addr;
-      new_args[nid] = cmb_entries[cid].begin_addr;
-      new_arg_sizes[nid] = (int64_t) ((char *)cmb_entries[cid].end_addr -
-          (char *)cmb_entries[cid].begin_addr);
-      new_arg_types[nid] = OMP_TGT_MAPTYPE_TARGET_PARAM;
-      DP("Entry %3d: base_addr " DPxMOD ", begin_addr " DPxMOD ", "
-          "size %" PRId64 ", type 0x%" PRIx64 "\n", nid,
-          DPxPTR(new_args_base[nid]), DPxPTR(new_args[nid]), new_arg_sizes[nid],
-          new_arg_types[nid]);
-    } else if (member_of[i] != -1) {
-      DP("Combined entry %3d has been encountered before, do nothing\n",
-          member_of[i]);
-    }
-
-    // Now that the combined entry (the one the old entry was a member of) has
-    // been inserted into the new arguments list, proceed with the old entry.
-    int nid = next_id++;
-    DP("Old entry %3d will become new entry %3d\n", i, nid);
-
-    new_args_base[nid] = args_base[i];
-    new_args[nid] = args[i];
-    new_arg_sizes[nid] = arg_sizes[i];
-    int64_t old_type = mod_arg_types[i];
-
-    if (is_ptr_old[i]) {
-      // Reset TO and FROM flags
-      old_type &= ~(OMP_TGT_OLDMAPTYPE_TO | OMP_TGT_OLDMAPTYPE_FROM);
-    }
-
-    if (member_of[i] == -1) {
-      if (!is_target_construct)
-        old_type &= ~OMP_TGT_MAPTYPE_TARGET_PARAM;
-      new_arg_types[nid] = old_type;
-      DP("Entry %3d: base_addr " DPxMOD ", begin_addr " DPxMOD ", size %" PRId64
-          ", type 0x%" PRIx64 " (old entry %d not MEMBER_OF)\n", nid,
-          DPxPTR(new_args_base[nid]), DPxPTR(new_args[nid]), new_arg_sizes[nid],
-          new_arg_types[nid], i);
-    } else {
-      // Old entry is not FIRST_MAP
-      old_type &= ~OMP_TGT_OLDMAPTYPE_FIRST_MAP;
-      // Add MEMBER_OF
-      int new_member_of = combined_to_new_id[member_of[i]];
-      old_type |= ((int64_t)new_member_of + 1) << 48;
-      new_arg_types[nid] = old_type;
-      DP("Entry %3d: base_addr " DPxMOD ", begin_addr " DPxMOD ", size %" PRId64
-        ", type 0x%" PRIx64 " (old entry %d MEMBER_OF %d)\n", nid,
-        DPxPTR(new_args_base[nid]), DPxPTR(new_args[nid]), new_arg_sizes[nid],
-        new_arg_types[nid], i, new_member_of);
-    }
-  }
-}
-
-static void cleanup_map(int32_t new_arg_num, void **new_args_base,
-    void **new_args, int64_t *new_arg_sizes, int64_t *new_arg_types,
-    int32_t arg_num, void **args_base) {
-  if (new_arg_num > 0) {
-    int offset = new_arg_num - arg_num;
-    for (int32_t i = 0; i < arg_num; ++i) {
-      // Restore old base address
-      args_base[i] = new_args_base[i+offset];
-    }
-    free(new_args_base);
-    free(new_args);
-    free(new_arg_sizes);
-    free(new_arg_types);
-  }
-}
-
 /// creates host-to-target data mapping, stores it in the
 /// libomptarget.so internal structure (an entry in a stack of data maps)
 /// and passes the data to the device.
 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
-  DP("Entering data begin region for device %ld with %d mappings\n", device_id,
-     arg_num);
+  if (IsOffloadDisabled()) return;
+
+  DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
+      device_id, arg_num);
 
   // No devices available?
   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
     device_id = omp_get_default_device();
-    DP("Use default device id %ld\n", device_id);
+    DP("Use default device id %" PRId64 "\n", device_id);
   }
 
   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
-    DP("Failed to get device %ld ready\n", device_id);
+    DP("Failed to get device %" PRId64 " ready\n", device_id);
+    HandleTargetOutcome(false);
     return;
   }
 
   DeviceTy& Device = Devices[device_id];
 
-  // Translate maps
-  int32_t new_arg_num;
-  void **new_args_base;
-  void **new_args;
-  int64_t *new_arg_sizes;
-  int64_t *new_arg_types;
-  translate_map(arg_num, args_base, args, arg_sizes, arg_types, new_arg_num,
-      new_args_base, new_args, new_arg_sizes, new_arg_types, false);
+#ifdef OMPTARGET_DEBUG
+  for (int i=0; i<arg_num; ++i) {
+    DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
+        ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
+        arg_sizes[i], arg_types[i]);
+  }
+#endif
 
-  //target_data_begin(Device, arg_num, args_base, args, arg_sizes, arg_types);
-  target_data_begin(Device, new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types);
-
-  // Cleanup translation memory
-  cleanup_map(new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types, arg_num, args_base);
+  int rc = target_data_begin(Device, arg_num, args_base,
+      args, arg_sizes, arg_types);
+  HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 }
 
 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
@@ -309,6 +134,7 @@ EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
 /// created by the last __tgt_target_data_begin.
 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  if (IsOffloadDisabled()) return;
   DP("Entering data end region with %d mappings\n", arg_num);
 
   // No devices available?
@@ -320,32 +146,29 @@ EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
   size_t Devices_size = Devices.size();
   RTLsMtx.unlock();
   if (Devices_size <= (size_t)device_id) {
-    DP("Device ID  %ld does not have a matching RTL.\n", device_id);
+    DP("Device ID  %" PRId64 " does not have a matching RTL.\n", device_id);
+    HandleTargetOutcome(false);
     return;
   }
 
   DeviceTy &Device = Devices[device_id];
   if (!Device.IsInit) {
-    DP("uninit device: ignore");
+    DP("Uninit device: ignore");
+    HandleTargetOutcome(false);
     return;
   }
 
-  // Translate maps
-  int32_t new_arg_num;
-  void **new_args_base;
-  void **new_args;
-  int64_t *new_arg_sizes;
-  int64_t *new_arg_types;
-  translate_map(arg_num, args_base, args, arg_sizes, arg_types, new_arg_num,
-      new_args_base, new_args, new_arg_sizes, new_arg_types, false);
+#ifdef OMPTARGET_DEBUG
+  for (int i=0; i<arg_num; ++i) {
+    DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
+        ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
+        arg_sizes[i], arg_types[i]);
+  }
+#endif
 
-  //target_data_end(Device, arg_num, args_base, args, arg_sizes, arg_types);
-  target_data_end(Device, new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types);
-
-  // Cleanup translation memory
-  cleanup_map(new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types, arg_num, args_base);
+  int rc = target_data_end(Device, arg_num, args_base,
+      args, arg_sizes, arg_types);
+  HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 }
 
 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
@@ -361,6 +184,7 @@ EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
 
 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  if (IsOffloadDisabled()) return;
   DP("Entering data update with %d mappings\n", arg_num);
 
   // No devices available?
@@ -369,12 +193,15 @@ EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
   }
 
   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
-    DP("Failed to get device %ld ready\n", device_id);
+    DP("Failed to get device %" PRId64 " ready\n", device_id);
+    HandleTargetOutcome(false);
     return;
   }
 
   DeviceTy& Device = Devices[device_id];
-  target_data_update(Device, arg_num, args_base, args, arg_sizes, arg_types);
+  int rc = target_data_update(Device, arg_num, args_base,
+      args, arg_sizes, arg_types);
+  HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 }
 
 EXTERN void __tgt_target_data_update_nowait(
@@ -390,36 +217,31 @@ EXTERN void __tgt_target_data_update_nowait(
 
 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
-  DP("Entering target region with entry point " DPxMOD " and device Id %ld\n",
-     DPxPTR(host_ptr), device_id);
+  if (IsOffloadDisabled()) return OFFLOAD_FAIL;
+  DP("Entering target region with entry point " DPxMOD " and device Id %"
+      PRId64 "\n", DPxPTR(host_ptr), device_id);
 
   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
     device_id = omp_get_default_device();
   }
 
   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
-    DP("Failed to get device %ld ready\n", device_id);
+    DP("Failed to get device %" PRId64 " ready\n", device_id);
+    HandleTargetOutcome(false);
     return OFFLOAD_FAIL;
   }
 
-  // Translate maps
-  int32_t new_arg_num;
-  void **new_args_base;
-  void **new_args;
-  int64_t *new_arg_sizes;
-  int64_t *new_arg_types;
-  translate_map(arg_num, args_base, args, arg_sizes, arg_types, new_arg_num,
-      new_args_base, new_args, new_arg_sizes, new_arg_types, true);
-
-  //return target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
-  //    arg_types, 0, 0, false /*team*/, false /*recursive*/);
-  int rc = target(device_id, host_ptr, new_arg_num, new_args_base, new_args,
-      new_arg_sizes, new_arg_types, 0, 0, false /*team*/);
-
-  // Cleanup translation memory
-  cleanup_map(new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types, arg_num, args_base);
+#ifdef OMPTARGET_DEBUG
+  for (int i=0; i<arg_num; ++i) {
+    DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
+        ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
+        arg_sizes[i], arg_types[i]);
+  }
+#endif
 
+  int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
+      arg_types, 0, 0, false /*team*/);
+  HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
   return rc;
 }
 
@@ -437,36 +259,31 @@ EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
-  DP("Entering target region with entry point " DPxMOD " and device Id %ld\n",
-     DPxPTR(host_ptr), device_id);
+  if (IsOffloadDisabled()) return OFFLOAD_FAIL;
+  DP("Entering target region with entry point " DPxMOD " and device Id %"
+      PRId64 "\n", DPxPTR(host_ptr), device_id);
 
   if (device_id == OFFLOAD_DEVICE_DEFAULT) {
     device_id = omp_get_default_device();
   }
 
   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
-    DP("Failed to get device %ld ready\n", device_id);
+    DP("Failed to get device %" PRId64 " ready\n", device_id);
+    HandleTargetOutcome(false);
     return OFFLOAD_FAIL;
   }
 
-  // Translate maps
-  int32_t new_arg_num;
-  void **new_args_base;
-  void **new_args;
-  int64_t *new_arg_sizes;
-  int64_t *new_arg_types;
-  translate_map(arg_num, args_base, args, arg_sizes, arg_types, new_arg_num,
-      new_args_base, new_args, new_arg_sizes, new_arg_types, true);
+#ifdef OMPTARGET_DEBUG
+  for (int i=0; i<arg_num; ++i) {
+    DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
+        ", Type=0x%" PRIx64 "\n", i, DPxPTR(args_base[i]), DPxPTR(args[i]),
+        arg_sizes[i], arg_types[i]);
+  }
+#endif
 
-  //return target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
-  //              arg_types, team_num, thread_limit, true /*team*/,
-  //              false /*recursive*/);
-  int rc = target(device_id, host_ptr, new_arg_num, new_args_base, new_args,
-      new_arg_sizes, new_arg_types, team_num, thread_limit, true /*team*/);
-
-  // Cleanup translation memory
-  cleanup_map(new_arg_num, new_args_base, new_args, new_arg_sizes,
-      new_arg_types, arg_num, args_base);
+  int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
+      arg_types, team_num, thread_limit, true /*team*/);
+  HandleTargetOutcome(rc == OFFLOAD_SUCCESS);
 
   return rc;
 }
@@ -491,11 +308,12 @@ EXTERN void __kmpc_push_target_tripcount(int64_t device_id,
   }
 
   if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
-    DP("Failed to get device %ld ready\n", device_id);
+    DP("Failed to get device %" PRId64 " ready\n", device_id);
+    HandleTargetOutcome(false);
     return;
   }
 
-  DP("__kmpc_push_target_tripcount(%ld, %" PRIu64 ")\n", device_id,
+  DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
       loop_tripcount);
   Devices[device_id].loopTripCnt = loop_tripcount;
 }

libomptarget/src/omptarget.cpp

@@ -25,6 +25,40 @@
 int DebugLevel = 0;
 #endif // OMPTARGET_DEBUG
 
+
+
+/* All begin addresses for partially mapped structs must be 8-aligned in order
+ * to ensure proper alignment of members. E.g.
+ *
+ * struct S {
+ *   int a;   // 4-aligned
+ *   int b;   // 4-aligned
+ *   int *p;  // 8-aligned
+ * } s1;
+ * ...
+ * #pragma omp target map(tofrom: s1.b, s1.p[0:N])
+ * {
+ *   s1.b = 5;
+ *   for (int i...) s1.p[i] = ...;
+ * }
+ *
+ * Here we are mapping s1 starting from member b, so BaseAddress=&s1=&s1.a and
+ * BeginAddress=&s1.b. Let's assume that the struct begins at address 0x100,
+ * then &s1.a=0x100, &s1.b=0x104, &s1.p=0x108. Each member obeys the alignment
+ * requirements for its type. Now, when we allocate memory on the device, in
+ * CUDA's case cuMemAlloc() returns an address which is at least 256-aligned.
+ * This means that the chunk of the struct on the device will start at a
+ * 256-aligned address, let's say 0x200. Then the address of b will be 0x200 and
+ * address of p will be a misaligned 0x204 (on the host there was no need to add
+ * padding between b and p, so p comes exactly 4 bytes after b). If the device
+ * kernel tries to access s1.p, a misaligned address error occurs (as reported
+ * by the CUDA plugin). By padding the begin address down to a multiple of 8 and
+ * extending the size of the allocated chuck accordingly, the chuck on the
+ * device will start at 0x200 with the padding (4 bytes), then &s1.b=0x204 and
+ * &s1.p=0x208, as they should be to satisfy the alignment requirements.
+ */
+static const int64_t alignment = 8;
+
 /// Map global data and execute pending ctors
 static int InitLibrary(DeviceTy& Device) {
   /*
@@ -172,7 +206,7 @@ int CheckDeviceAndCtors(int64_t device_id) {
   return OFFLOAD_SUCCESS;
 }
 
-static short member_of(int64_t type) {
+static int32_t member_of(int64_t type) {
   return ((type & OMP_TGT_MAPTYPE_MEMBER_OF) >> 48) - 1;
 }
 
@@ -180,7 +214,6 @@ static short member_of(int64_t type) {
 int target_data_begin(DeviceTy &Device, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
   // process each input.
-  int rc = OFFLOAD_SUCCESS;
   for (int32_t i = 0; i < arg_num; ++i) {
     // Ignore private variables and arrays - there is no mapping for them.
     if ((arg_types[i] & OMP_TGT_MAPTYPE_LITERAL) ||
@@ -189,10 +222,33 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
 
     void *HstPtrBegin = args[i];
     void *HstPtrBase = args_base[i];
+    int64_t data_size = arg_sizes[i];
+
+    // Adjust for proper alignment if this is a combined entry (for structs).
+    // Look at the next argument - if that is MEMBER_OF this one, then this one
+    // is a combined entry.
+    int64_t padding = 0;
+    const int next_i = i+1;
+    if (member_of(arg_types[i]) < 0 && next_i < arg_num &&
+        member_of(arg_types[next_i]) == i) {
+      padding = (int64_t)HstPtrBegin % alignment;
+      if (padding) {
+        DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
+            "\n", padding, DPxPTR(HstPtrBegin));
+        HstPtrBegin = (char *) HstPtrBegin - padding;
+        data_size += padding;
+      }
+    }
+
     // Address of pointer on the host and device, respectively.
     void *Pointer_HstPtrBegin, *Pointer_TgtPtrBegin;
     bool IsNew, Pointer_IsNew;
     bool IsImplicit = arg_types[i] & OMP_TGT_MAPTYPE_IMPLICIT;
+    // UpdateRef is based on MEMBER_OF instead of TARGET_PARAM because if we
+    // have reached this point via __tgt_target_data_begin and not __tgt_target
+    // then no argument is marked as TARGET_PARAM ("omp target data map" is not
+    // associated with a target region, so there are no target parameters). This
+    // may be considered a hack, we could revise the scheme in the future.
     bool UpdateRef = !(arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF);
     if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
       DP("Has a pointer entry: \n");
@@ -202,6 +258,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
       if (!Pointer_TgtPtrBegin) {
         DP("Call to getOrAllocTgtPtr returned null pointer (device failure or "
             "illegal mapping).\n");
+        return OFFLOAD_FAIL;
       }
       DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new"
           "\n", sizeof(void *), DPxPTR(Pointer_TgtPtrBegin),
@@ -213,28 +270,22 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
     }
 
     void *TgtPtrBegin = Device.getOrAllocTgtPtr(HstPtrBegin, HstPtrBase,
-        arg_sizes[i], IsNew, IsImplicit, UpdateRef);
-    if (!TgtPtrBegin && arg_sizes[i]) {
-      // If arg_sizes[i]==0, then the argument is a pointer to NULL, so
-      // getOrAlloc() returning NULL is not an error.
+        data_size, IsNew, IsImplicit, UpdateRef);
+    if (!TgtPtrBegin && data_size) {
+      // If data_size==0, then the argument could be a zero-length pointer to
+      // NULL, so getOrAlloc() returning NULL is not an error.
       DP("Call to getOrAllocTgtPtr returned null pointer (device failure or "
           "illegal mapping).\n");
     }
     DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
-        " - is%s new\n", arg_sizes[i], DPxPTR(TgtPtrBegin),
+        " - is%s new\n", data_size, DPxPTR(TgtPtrBegin),
         (IsNew ? "" : " not"));
 
     if (arg_types[i] & OMP_TGT_MAPTYPE_RETURN_PARAM) {
-      void *ret_ptr;
-      if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
-        ret_ptr = Pointer_TgtPtrBegin;
-      else {
-        bool IsLast; // not used
-        ret_ptr = Device.getTgtPtrBegin(HstPtrBegin, 0, IsLast, false);
-      }
-
-      DP("Returning device pointer " DPxMOD "\n", DPxPTR(ret_ptr));
-      args_base[i] = ret_ptr;
+      uintptr_t Delta = (uintptr_t)HstPtrBegin - (uintptr_t)HstPtrBase;
+      void *TgtPtrBase = (void *)((uintptr_t)TgtPtrBegin - Delta);
+      DP("Returning device pointer " DPxMOD "\n", DPxPTR(TgtPtrBase));
+      args_base[i] = TgtPtrBase;
     }
 
     if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
@@ -243,7 +294,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
         copy = true;
       } else if (arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) {
         // Copy data only if the "parent" struct has RefCount==1.
-        short parent_idx = member_of(arg_types[i]);
+        int32_t parent_idx = member_of(arg_types[i]);
         long parent_rc = Device.getMapEntryRefCnt(args[parent_idx]);
         assert(parent_rc > 0 && "parent struct not found");
         if (parent_rc == 1) {
@@ -253,11 +304,11 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
 
       if (copy) {
         DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n",
-            arg_sizes[i], DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin));
-        int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, arg_sizes[i]);
+            data_size, DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin));
+        int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, data_size);
         if (rt != OFFLOAD_SUCCESS) {
           DP("Copying data to device failed.\n");
-          rc = OFFLOAD_FAIL;
+          return OFFLOAD_FAIL;
         }
       }
     }
@@ -271,7 +322,7 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
           sizeof(void *));
       if (rt != OFFLOAD_SUCCESS) {
         DP("Copying data to device failed.\n");
-        rc = OFFLOAD_FAIL;
+        return OFFLOAD_FAIL;
       }
       // create shadow pointers for this entry
       Device.ShadowMtx.lock();
@@ -281,13 +332,12 @@ int target_data_begin(DeviceTy &Device, int32_t arg_num,
     }
   }
 
-  return rc;
+  return OFFLOAD_SUCCESS;
 }
 
 /// Internal function to undo the mapping and retrieve the data from the device.
 int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
     void **args, int64_t *arg_sizes, int64_t *arg_types) {
-  int rc = OFFLOAD_SUCCESS;
   // process each input.
   for (int32_t i = arg_num - 1; i >= 0; --i) {
     // Ignore private variables and arrays - there is no mapping for them.
@@ -297,16 +347,33 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
       continue;
 
     void *HstPtrBegin = args[i];
+    int64_t data_size = arg_sizes[i];
+    // Adjust for proper alignment if this is a combined entry (for structs).
+    // Look at the next argument - if that is MEMBER_OF this one, then this one
+    // is a combined entry.
+    int64_t padding = 0;
+    const int next_i = i+1;
+    if (member_of(arg_types[i]) < 0 && next_i < arg_num &&
+        member_of(arg_types[next_i]) == i) {
+      padding = (int64_t)HstPtrBegin % alignment;
+      if (padding) {
+        DP("Using a padding of %" PRId64 " bytes for begin address " DPxMOD
+            "\n", padding, DPxPTR(HstPtrBegin));
+        HstPtrBegin = (char *) HstPtrBegin - padding;
+        data_size += padding;
+      }
+    }
+
     bool IsLast;
     bool UpdateRef = !(arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) ||
         (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ);
     bool ForceDelete = arg_types[i] & OMP_TGT_MAPTYPE_DELETE;
 
     // If PTR_AND_OBJ, HstPtrBegin is address of pointee
-    void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, arg_sizes[i], IsLast,
+    void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, data_size, IsLast,
         UpdateRef);
     DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
-        " - is%s last\n", arg_sizes[i], DPxPTR(TgtPtrBegin),
+        " - is%s last\n", data_size, DPxPTR(TgtPtrBegin),
         (IsLast ? "" : " not"));
 
     bool DelEntry = IsLast || ForceDelete;
@@ -324,7 +391,7 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
         if ((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
             !(arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) {
           // Copy data only if the "parent" struct has RefCount==1.
-          short parent_idx = member_of(arg_types[i]);
+          int32_t parent_idx = member_of(arg_types[i]);
           long parent_rc = Device.getMapEntryRefCnt(args[parent_idx]);
           assert(parent_rc > 0 && "parent struct not found");
           if (parent_rc == 1) {
@@ -334,11 +401,11 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
 
         if (DelEntry || Always || CopyMember) {
           DP("Moving %" PRId64 " bytes (tgt:" DPxMOD ") -> (hst:" DPxMOD ")\n",
-              arg_sizes[i], DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
-          int rt = Device.data_retrieve(HstPtrBegin, TgtPtrBegin, arg_sizes[i]);
+              data_size, DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
+          int rt = Device.data_retrieve(HstPtrBegin, TgtPtrBegin, data_size);
           if (rt != OFFLOAD_SUCCESS) {
             DP("Copying data from device failed.\n");
-            rc = OFFLOAD_FAIL;
+            return OFFLOAD_FAIL;
           }
         }
       }
@@ -348,16 +415,18 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
       // copies. If the struct is going to be deallocated, remove any remaining
       // shadow pointer entries for this struct.
       uintptr_t lb = (uintptr_t) HstPtrBegin;
-      uintptr_t ub = (uintptr_t) HstPtrBegin + arg_sizes[i];
+      uintptr_t ub = (uintptr_t) HstPtrBegin + data_size;
       Device.ShadowMtx.lock();
       for (ShadowPtrListTy::iterator it = Device.ShadowPtrMap.begin();
-          it != Device.ShadowPtrMap.end(); ++it) {
+           it != Device.ShadowPtrMap.end();) {
         void **ShadowHstPtrAddr = (void**) it->first;
 
         // An STL map is sorted on its keys; use this property
         // to quickly determine when to break out of the loop.
-        if ((uintptr_t) ShadowHstPtrAddr < lb)
+        if ((uintptr_t) ShadowHstPtrAddr < lb) {
+          ++it;
           continue;
+        }
         if ((uintptr_t) ShadowHstPtrAddr >= ub)
           break;
 
@@ -371,27 +440,29 @@ int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
         // If the struct is to be deallocated, remove the shadow entry.
         if (DelEntry) {
           DP("Removing shadow pointer " DPxMOD "\n", DPxPTR(ShadowHstPtrAddr));
-          Device.ShadowPtrMap.erase(it);
+          it = Device.ShadowPtrMap.erase(it);
+        } else {
+          ++it;
         }
       }
       Device.ShadowMtx.unlock();
 
       // Deallocate map
       if (DelEntry) {
-        int rt = Device.deallocTgtPtr(HstPtrBegin, arg_sizes[i], ForceDelete);
+        int rt = Device.deallocTgtPtr(HstPtrBegin, data_size, ForceDelete);
         if (rt != OFFLOAD_SUCCESS) {
           DP("Deallocating data from device failed.\n");
-          rc = OFFLOAD_FAIL;
+          return OFFLOAD_FAIL;
         }
       }
     }
   }
 
-  return rc;
+  return OFFLOAD_SUCCESS;
 }
 
 /// Internal function to pass data to/from the target.
-void target_data_update(DeviceTy &Device, int32_t arg_num,
+int target_data_update(DeviceTy &Device, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
   // process each input.
   for (int32_t i = 0; i < arg_num; ++i) {
@@ -404,11 +475,19 @@ void target_data_update(DeviceTy &Device, int32_t arg_num,
     bool IsLast;
     void *TgtPtrBegin = Device.getTgtPtrBegin(HstPtrBegin, MapSize, IsLast,
         false);
+    if (!TgtPtrBegin) {
+      DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
+      continue;
+    }
 
     if (arg_types[i] & OMP_TGT_MAPTYPE_FROM) {
       DP("Moving %" PRId64 " bytes (tgt:" DPxMOD ") -> (hst:" DPxMOD ")\n",
           arg_sizes[i], DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin));
-      Device.data_retrieve(HstPtrBegin, TgtPtrBegin, MapSize);
+      int rt = Device.data_retrieve(HstPtrBegin, TgtPtrBegin, MapSize);
+      if (rt != OFFLOAD_SUCCESS) {
+        DP("Copying data from device failed.\n");
+        return OFFLOAD_FAIL;
+      }
 
       uintptr_t lb = (uintptr_t) HstPtrBegin;
       uintptr_t ub = (uintptr_t) HstPtrBegin + MapSize;
@@ -431,8 +510,11 @@ void target_data_update(DeviceTy &Device, int32_t arg_num,
     if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
       DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n",
           arg_sizes[i], DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBegin));
-      Device.data_submit(TgtPtrBegin, HstPtrBegin, MapSize);
-
+      int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, MapSize);
+      if (rt != OFFLOAD_SUCCESS) {
+        DP("Copying data to device failed.\n");
+        return OFFLOAD_FAIL;
+      }
       uintptr_t lb = (uintptr_t) HstPtrBegin;
       uintptr_t ub = (uintptr_t) HstPtrBegin + MapSize;
       Device.ShadowMtx.lock();
@@ -446,12 +528,25 @@ void target_data_update(DeviceTy &Device, int32_t arg_num,
         DP("Restoring original target pointer value " DPxMOD " for target "
             "pointer " DPxMOD "\n", DPxPTR(it->second.TgtPtrVal),
             DPxPTR(it->second.TgtPtrAddr));
-        Device.data_submit(it->second.TgtPtrAddr,
+        rt = Device.data_submit(it->second.TgtPtrAddr,
             &it->second.TgtPtrVal, sizeof(void *));
+        if (rt != OFFLOAD_SUCCESS) {
+          DP("Copying data to device failed.\n");
+          Device.ShadowMtx.unlock();
+          return OFFLOAD_FAIL;
+        }
       }
       Device.ShadowMtx.unlock();
     }
   }
+  return OFFLOAD_SUCCESS;
+}
+
+static const unsigned LambdaMapping = OMP_TGT_MAPTYPE_PTR_AND_OBJ |
+                                      OMP_TGT_MAPTYPE_LITERAL |
+                                      OMP_TGT_MAPTYPE_IMPLICIT;
+static bool isLambdaMapping(int64_t Mapping) {
+  return (Mapping & LambdaMapping) == LambdaMapping;
 }
 
 /// performs the same actions as data_begin in case arg_num is
@@ -519,12 +614,8 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
   // Move data to device.
   int rc = target_data_begin(Device, arg_num, args_base, args, arg_sizes,
       arg_types);
-
   if (rc != OFFLOAD_SUCCESS) {
-    DP("Call to target_data_begin failed, skipping target execution.\n");
-    // Call target_data_end to dealloc whatever target_data_begin allocated
-    // and return OFFLOAD_FAIL.
-    target_data_end(Device, arg_num, args_base, args, arg_sizes, arg_types);
+    DP("Call to target_data_begin failed, abort target.\n");
     return OFFLOAD_FAIL;
   }
 
@@ -533,10 +624,45 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
 
   // List of (first-)private arrays allocated for this target region
   std::vector<void *> fpArrays;
+  std::vector<int> tgtArgsPositions(arg_num, -1);
 
   for (int32_t i = 0; i < arg_num; ++i) {
     if (!(arg_types[i] & OMP_TGT_MAPTYPE_TARGET_PARAM)) {
       // This is not a target parameter, do not push it into tgt_args.
+      // Check for lambda mapping.
+      if (isLambdaMapping(arg_types[i])) {
+        assert((arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) &&
+               "PTR_AND_OBJ must be also MEMBER_OF.");
+        unsigned idx = member_of(arg_types[i]);
+        int tgtIdx = tgtArgsPositions[idx];
+        assert(tgtIdx != -1 && "Base address must be translated already.");
+        // The parent lambda must be processed already and it must be the last
+        // in tgt_args and tgt_offsets arrays.
+        void *HstPtrVal = args[i];
+        void *HstPtrBegin = args_base[i];
+        void *HstPtrBase = args[idx];
+        bool IsLast; // unused.
+        void *TgtPtrBase =
+            (void *)((intptr_t)tgt_args[tgtIdx] + tgt_offsets[tgtIdx]);
+        DP("Parent lambda base " DPxMOD "\n", DPxPTR(TgtPtrBase));
+        uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
+        void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta);
+        void *Pointer_TgtPtrBegin =
+            Device.getTgtPtrBegin(HstPtrVal, arg_sizes[i], IsLast, false);
+        if (!Pointer_TgtPtrBegin) {
+          DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n",
+             DPxPTR(HstPtrVal));
+          continue;
+        }
+        DP("Update lambda reference (" DPxMOD ") -> [" DPxMOD "]\n",
+           DPxPTR(Pointer_TgtPtrBegin), DPxPTR(TgtPtrBegin));
+        int rt = Device.data_submit(TgtPtrBegin, &Pointer_TgtPtrBegin,
+                                    sizeof(void *));
+        if (rt != OFFLOAD_SUCCESS) {
+          DP("Copying data to device failed.\n");
+          return OFFLOAD_FAIL;
+        }
+      }
       continue;
     }
     void *HstPtrBegin = args[i];
@@ -554,30 +680,28 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
       TgtPtrBegin = Device.RTL->data_alloc(Device.RTLDeviceID,
           arg_sizes[i], HstPtrBegin);
       if (!TgtPtrBegin) {
-        DP ("Data allocation for %sprivate array " DPxMOD " failed\n",
+        DP ("Data allocation for %sprivate array " DPxMOD " failed, "
+            "abort target.\n",
             (arg_types[i] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
             DPxPTR(HstPtrBegin));
-        rc = OFFLOAD_FAIL;
-        break;
-      } else {
-        fpArrays.push_back(TgtPtrBegin);
-        TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
+        return OFFLOAD_FAIL;
+      }
+      fpArrays.push_back(TgtPtrBegin);
+      TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
 #ifdef OMPTARGET_DEBUG
-        void *TgtPtrBase = (void *)((intptr_t)TgtPtrBegin + TgtBaseOffset);
-        DP("Allocated %" PRId64 " bytes of target memory at " DPxMOD " for "
-            "%sprivate array " DPxMOD " - pushing target argument " DPxMOD "\n",
-            arg_sizes[i], DPxPTR(TgtPtrBegin),
-            (arg_types[i] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
-            DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBase));
+      void *TgtPtrBase = (void *)((intptr_t)TgtPtrBegin + TgtBaseOffset);
+      DP("Allocated %" PRId64 " bytes of target memory at " DPxMOD " for "
+          "%sprivate array " DPxMOD " - pushing target argument " DPxMOD "\n",
+          arg_sizes[i], DPxPTR(TgtPtrBegin),
+          (arg_types[i] & OMP_TGT_MAPTYPE_TO ? "first-" : ""),
+          DPxPTR(HstPtrBegin), DPxPTR(TgtPtrBase));
 #endif
-        // If first-private, copy data from host
-        if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
-          int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, arg_sizes[i]);
-          if (rt != OFFLOAD_SUCCESS) {
-            DP ("Copying data to device failed.\n");
-            rc = OFFLOAD_FAIL;
-            break;
-          }
+      // If first-private, copy data from host
+      if (arg_types[i] & OMP_TGT_MAPTYPE_TO) {
+        int rt = Device.data_submit(TgtPtrBegin, HstPtrBegin, arg_sizes[i]);
+        if (rt != OFFLOAD_SUCCESS) {
+          DP ("Copying data to device failed, failed.\n");
+          return OFFLOAD_FAIL;
         }
       }
     } else if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
@@ -597,6 +721,7 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
           DPxPTR(TgtPtrBase), DPxPTR(HstPtrBegin));
 #endif
     }
+    tgtArgsPositions[i] = tgt_args.size();
     tgt_args.push_back(TgtPtrBegin);
     tgt_offsets.push_back(TgtBaseOffset);
   }
@@ -609,21 +734,20 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
   Device.loopTripCnt = 0;
 
   // Launch device execution.
-  if (rc == OFFLOAD_SUCCESS) {
-    DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",
-        TargetTable->EntriesBegin[TM->Index].name,
-        DPxPTR(TargetTable->EntriesBegin[TM->Index].addr), TM->Index);
-    if (IsTeamConstruct) {
-      rc = Device.run_team_region(TargetTable->EntriesBegin[TM->Index].addr,
-          &tgt_args[0], &tgt_offsets[0], tgt_args.size(), team_num,
-          thread_limit, ltc);
-    } else {
-      rc = Device.run_region(TargetTable->EntriesBegin[TM->Index].addr,
-          &tgt_args[0], &tgt_offsets[0], tgt_args.size());
-    }
+  DP("Launching target execution %s with pointer " DPxMOD " (index=%d).\n",
+      TargetTable->EntriesBegin[TM->Index].name,
+      DPxPTR(TargetTable->EntriesBegin[TM->Index].addr), TM->Index);
+  if (IsTeamConstruct) {
+    rc = Device.run_team_region(TargetTable->EntriesBegin[TM->Index].addr,
+        &tgt_args[0], &tgt_offsets[0], tgt_args.size(), team_num,
+        thread_limit, ltc);
   } else {
-    DP("Errors occurred while obtaining target arguments, skipping kernel "
-        "execution\n");
+    rc = Device.run_region(TargetTable->EntriesBegin[TM->Index].addr,
+        &tgt_args[0], &tgt_offsets[0], tgt_args.size());
+  }
+  if (rc != OFFLOAD_SUCCESS) {
+    DP ("Executing target region abort target.\n");
+    return OFFLOAD_FAIL;
   }
 
   // Deallocate (first-)private arrays
@@ -631,18 +755,17 @@ int target(int64_t device_id, void *host_ptr, int32_t arg_num,
     int rt = Device.RTL->data_delete(Device.RTLDeviceID, it);
     if (rt != OFFLOAD_SUCCESS) {
       DP("Deallocation of (first-)private arrays failed.\n");
-      rc = OFFLOAD_FAIL;
+      return OFFLOAD_FAIL;
     }
   }
 
   // Move data from device.
   int rt = target_data_end(Device, arg_num, args_base, args, arg_sizes,
       arg_types);
-
   if (rt != OFFLOAD_SUCCESS) {
-    DP("Call to target_data_end failed.\n");
-    rc = OFFLOAD_FAIL;
+    DP("Call to target_data_end failed, abort targe.\n");
+    return OFFLOAD_FAIL;
   }
 
-  return rc;
+  return OFFLOAD_SUCCESS;
 }

libomptarget/src/private.h

@@ -24,7 +24,7 @@ extern int target_data_begin(DeviceTy &Device, int32_t arg_num,
 extern int target_data_end(DeviceTy &Device, int32_t arg_num, void **args_base,
     void **args, int64_t *arg_sizes, int64_t *arg_types);
 
-extern void target_data_update(DeviceTy &Device, int32_t arg_num,
+extern int target_data_update(DeviceTy &Device, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types);
 
 extern int target(int64_t device_id, void *host_ptr, int32_t arg_num,
@@ -33,6 +33,44 @@ extern int target(int64_t device_id, void *host_ptr, int32_t arg_num,
 
 extern int CheckDeviceAndCtors(int64_t device_id);
 
+// enum for OMP_TARGET_OFFLOAD; keep in sync with kmp.h definition
+enum kmp_target_offload_kind {
+  tgt_disabled = 0,
+  tgt_default = 1,
+  tgt_mandatory = 2
+};
+typedef enum kmp_target_offload_kind kmp_target_offload_kind_t;
+extern kmp_target_offload_kind_t TargetOffloadPolicy;
+
+////////////////////////////////////////////////////////////////////////////////
+// implemtation for fatal messages
+////////////////////////////////////////////////////////////////////////////////
+
+#define FATAL_MESSAGE0(_num, _str)                                    \
+  do {                                                                \
+    fprintf(stderr, "Libomptarget fatal error %d: %s\n", _num, _str); \
+    exit(1);                                                          \
+  } while (0)
+
+#define FATAL_MESSAGE(_num, _str, ...)                              \
+  do {                                                              \
+    fprintf(stderr, "Libomptarget fatal error %d:" _str "\n", _num, \
+            __VA_ARGS__);                                           \
+    exit(1);                                                        \
+  } while (0)
+
+// Implemented in libomp, they are called from within __tgt_* functions.
+#ifdef __cplusplus
+extern "C" {
+#endif
+// functions that extract info from libomp; keep in sync
+int omp_get_default_device(void) __attribute__((weak));
+int32_t __kmpc_omp_taskwait(void *loc_ref, int32_t gtid) __attribute__((weak));
+int __kmpc_get_target_offload(void) __attribute__((weak));
+#ifdef __cplusplus
+}
+#endif
+
 #ifdef OMPTARGET_DEBUG
 extern int DebugLevel;
 

libomptarget/src/rtl.cpp

@@ -16,9 +16,11 @@
 #include "rtl.h"
 
 #include <cassert>
+#include <cstdlib>
 #include <cstring>
 #include <dlfcn.h>
 #include <mutex>
+#include <string>
 
 // List of all plugins that can support offloading.
 static const char *RTLNames[] = {
@@ -44,9 +46,8 @@ void RTLsTy::LoadRTLs() {
 #endif // OMPTARGET_DEBUG
 
   // Parse environment variable OMP_TARGET_OFFLOAD (if set)
-  char *envStr = getenv("OMP_TARGET_OFFLOAD");
-  if (envStr && !strcmp(envStr, "DISABLED")) {
-    DP("Target offloading disabled by environment\n");
+  TargetOffloadPolicy = (kmp_target_offload_kind_t) __kmpc_get_target_offload();
+  if (TargetOffloadPolicy == tgt_disabled) {
     return;
   }
 
@@ -214,7 +215,6 @@ void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
       if (!R.isUsed) {
         // Initialize the device information for the RTL we are about to use.
         DeviceTy device(&R);
-
         size_t start = Devices.size();
         Devices.resize(start + R.NumberOfDevices, device);
         for (int32_t device_id = 0; device_id < R.NumberOfDevices;
@@ -223,9 +223,6 @@ void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
           Devices[start + device_id].DeviceID = start + device_id;
           // RTL local device ID
           Devices[start + device_id].RTLDeviceID = device_id;
-
-          // Save pointer to device in RTL in case we want to unregister the RTL
-          R.Devices.push_back(&Devices[start + device_id]);
         }
 
         // Initialize the index of this RTL and save it in the used RTLs.

libomptarget/src/rtl.h

@@ -17,6 +17,7 @@
 #include <list>
 #include <map>
 #include <mutex>
+#include <string>
 #include <vector>
 
 // Forward declarations.
@@ -42,7 +43,6 @@ struct RTLInfoTy {
                                    // i.e. the OpenMP index of the first device
                                    // to be registered with this RTL.
   int32_t NumberOfDevices;         // Number of devices this RTL deals with.
-  std::vector<DeviceTy *> Devices; // one per device (NumberOfDevices in total).
 
   void *LibraryHandler;
 
@@ -73,7 +73,7 @@ struct RTLInfoTy {
   // The existence of the mutex above makes RTLInfoTy non-copyable.
   // We need to provide a copy constructor explicitly.
   RTLInfoTy()
-      : Idx(-1), NumberOfDevices(-1), Devices(), LibraryHandler(0),
+      : Idx(-1), NumberOfDevices(-1), LibraryHandler(0),
 #ifdef OMPTARGET_DEBUG
         RTLName(),
 #endif
@@ -85,7 +85,6 @@ struct RTLInfoTy {
   RTLInfoTy(const RTLInfoTy &r) : Mtx() {
     Idx = r.Idx;
     NumberOfDevices = r.NumberOfDevices;
-    Devices = r.Devices;
     LibraryHandler = r.LibraryHandler;
 #ifdef OMPTARGET_DEBUG
     RTLName = r.RTLName;

libomptarget/test/CMakeLists.txt

@@ -1,6 +1,6 @@
 # CMakeLists.txt file for unit testing OpenMP offloading runtime library.
-if(NOT "${OPENMP_TEST_COMPILER_ID}" STREQUAL "Clang" OR
-   ${OPENMP_TEST_COMPILER_VERSION} VERSION_LESS 6.0.0)
+if(NOT OPENMP_TEST_COMPILER_ID STREQUAL "Clang" OR
+   OPENMP_TEST_COMPILER_VERSION VERSION_LESS 6.0.0)
   libomptarget_say("Can only test with Clang compiler in version 6.0.0 or later.")
   libomptarget_warning_say("The check-libomptarget target will not be available!")
   return()
@@ -14,15 +14,6 @@ endif()
 
 add_openmp_testsuite(check-libomptarget "Running libomptarget tests" ${CMAKE_CURRENT_BINARY_DIR} DEPENDS omptarget omp)
 
-if(${OPENMP_STANDALONE_BUILD})
-  set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/../../runtime/src" CACHE STRING
-    "Path to folder containing omp.h")
-  set(LIBOMPTARGET_OPENMP_HOST_RTL_FOLDER "${CMAKE_CURRENT_BINARY_DIR}/../../runtime/src" CACHE STRING
-    "Path to folder containing libomp.so")
-else()
-  set(LIBOMPTARGET_OPENMP_HEADER_FOLDER "${LIBOMPTARGET_BINARY_DIR}/../runtime/src")
-endif()
-
 # Configure the lit.site.cfg.in file
 set(AUTO_GEN_COMMENT "## Autogenerated by libomptarget configuration.\n# Do not edit!")
 configure_file(lit.site.cfg.in lit.site.cfg @ONLY)

libomptarget/test/api/omp_get_num_devices.c

@@ -0,0 +1,36 @@
+// RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+
+#include <stdio.h>
+#include <omp.h>
+
+int test_omp_get_num_devices()
+{
+  /* checks that omp_get_num_devices() > 0 */
+  int num_devices = omp_get_num_devices();
+  printf("num_devices = %d\n", num_devices);
+
+  #pragma omp target
+  {}
+
+  return (num_devices > 0);
+}
+
+int main()
+{
+  int i;
+  int failed=0;
+
+  if (!test_omp_get_num_devices()) {
+    failed++;
+  }
+  if (failed)
+    printf("FAIL\n");
+  else
+    printf("PASS\n");
+  return failed;
+}
+
+// CHECK: PASS

libomptarget/test/lit.cfg

@@ -41,6 +41,11 @@ if config.omp_host_rtl_directory:
 
 config.test_flags = config.test_flags + " " + config.test_extra_flags
 
+# Allow REQUIRES / UNSUPPORTED / XFAIL to work
+config.target_triple = [ ]
+for feature in config.test_compiler_features:
+    config.available_features.add(feature)
+
 if config.libomptarget_debug:
   config.available_features.add('libomptarget-debug')
 

libomptarget/test/lit.site.cfg.in

@@ -2,6 +2,7 @@
 
 config.test_c_compiler = "@OPENMP_TEST_C_COMPILER@"
 config.test_cxx_compiler = "@OPENMP_TEST_CXX_COMPILER@"
+config.test_compiler_features = @OPENMP_TEST_COMPILER_FEATURES@
 config.test_openmp_flags = "@OPENMP_TEST_OPENMP_FLAGS@"
 config.test_extra_flags = "@OPENMP_TEST_FLAGS@"
 config.libomptarget_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
@@ -15,4 +16,4 @@ config.libomptarget_filecheck = "@OPENMP_FILECHECK_EXECUTABLE@"
 config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
 
 # Let the main config do the real work.
-lit_config.load_config(config, "@LIBOMPTARGET_BASE_DIR@/test/lit.cfg")
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")

libomptarget/test/mapping/pr38704.c

@@ -0,0 +1,47 @@
+// RUN: %libomptarget-compile-run-and-check-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compile-run-and-check-x86_64-pc-linux-gnu
+
+// Clang 6.0 doesn't use the new map interface, undefined behavior when
+// the compiler emits "old" interface code for structures.
+// UNSUPPORTED: clang-6
+
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+  int *ptr1;
+  int *ptr2;
+} StructWithPtrs;
+
+int main(int argc, char *argv[]) {
+  StructWithPtrs s, s2;
+  s.ptr1 = malloc(sizeof(int));
+  s.ptr2 = malloc(2 * sizeof(int));
+  s2.ptr1 = malloc(sizeof(int));
+  s2.ptr2 = malloc(2 * sizeof(int));
+
+#pragma omp target enter data map(to: s2.ptr2[0:1])
+#pragma omp target map(s.ptr1[0:1], s.ptr2[0:2])
+  {
+    s.ptr1[0] = 1;
+    s.ptr2[0] = 2;
+    s.ptr2[1] = 3;
+  }
+#pragma omp target exit data map(from: s2.ptr1[0:1], s2.ptr2[0:1])
+
+  // CHECK: s.ptr1[0] = 1
+  // CHECK: s.ptr2[0] = 2
+  // CHECK: s.ptr2[1] = 3
+  printf("s.ptr1[0] = %d\n", s.ptr1[0]);
+  printf("s.ptr2[0] = %d\n", s.ptr2[0]);
+  printf("s.ptr2[1] = %d\n", s.ptr2[1]);
+
+  free(s.ptr1);
+  free(s.ptr2);
+  free(s2.ptr1);
+  free(s2.ptr2);
+
+  return 0;
+}

offload/Makefile

@@ -1,224 +0,0 @@
-#
-##//===----------------------------------------------------------------------===//
-#//
-#//                     The LLVM Compiler Infrastructure
-#//
-#// This file is dual licensed under the MIT and the University of Illinois Open
-#// Source Licenses. See LICENSE.txt for details.
-#//
-#//===----------------------------------------------------------------------===//
-#
-
-# MAKEFILE PARAMETERS
-#
-# root_dir - path to root directory of liboffload
-# build_dir - path to build directory
-# mpss_dir - path to root directory of mpss
-# mpss_version - version of the mpss (e.g., version "3.3.x" would be "33")
-# libiomp_host_dir - path to host libiomp directory (unnecessary if compiler_host is icc)
-# libiomp_target_dir - path to target libiomp directory (unnecesarry if compiler_target is icc)
-# omp_header_dir - path to omp.h (unnecessary if compiler_host and compiler_target are icc)
-# os_host - host operating system
-# os_target - target operating system
-# compiler_host - host compiler
-# compiler_target - target compiler
-# options_host - additional options for host compiler
-# options_target - additional options for target compiler
-#
-
-# Directories
-root_dir?=.
-build_dir?=$(root_dir)/build
-build_host_dir=$(build_dir)/host
-build_target_dir=$(build_dir)/target
-obj_host_dir=$(build_dir)/obj_host
-obj_target_dir=$(build_dir)/obj_target
-source_dir=$(root_dir)/src
-imported_dir=$(source_dir)/imported
-
-# OS
-os_host?=linux
-os_target?=linux
-ifneq ($(os_host)_$(os_target), linux_linux)
-  $(error "Only linux is supported")
-endif
-
-# Compilers
-compiler_host?=gcc
-compiler_target?=gcc
-
-# MPSS
-mpss_version?=30
-mpss_dir?=/
-mpss_present=$(shell if test -d $(mpss_dir); then echo OK; else echo KO; fi)
-ifneq ($(mpss_present), OK)
-  $(error "Cannot find MPSS directory $(mpss_dir)")
-endif
-
-ifeq ($(shell test $(mpss_version) -gt 33; echo $$?), 0)
-  coi_dir=$(mpss_dir)/sysroots/k1om-mpss-linux/usr
-  coi_include=$(coi_dir)/include/intel-coi
-  coi_lib_host=$(mpss_dir)/lib64
-  coi_lib_device=$(coi_dir)/lib64
-else
-  coi_dir=$(mpss_dir)/opt/intel/mic/coi
-  coi_include=$(coi_dir)/include
-  coi_lib_host=$(coi_dir)/host-linux-release/lib
-  coi_lib_device=$(coi_dir)/device-linux-release/lib
-endif
-myo_dir=$(mpss_dir)/opt/intel/mic/myo
-
-# Sources
-src_liboffload_common=dv_util.cpp liboffload_error.c liboffload_msg.c offload_common.cpp offload_table.cpp offload_trace.cpp offload_util.cpp
-
-src_liboffload_host=$(src_liboffload_common) cean_util.cpp coi/coi_client.cpp compiler_if_host.cpp offload_engine.cpp offload_env.cpp offload_host.cpp offload_omp_host.cpp offload_timer_host.cpp offload_orsl.cpp orsl-lite/lib/orsl-lite.c offload_myo_host.cpp
-src_liboffload_host:=$(foreach file,$(src_liboffload_host),$(source_dir)/$(file))
-
-src_liboffload_target=$(src_liboffload_common) coi/coi_server.cpp compiler_if_target.cpp offload_omp_target.cpp offload_target.cpp offload_timer_target.cpp offload_myo_target.cpp
-src_liboffload_target:=$(foreach file,$(src_liboffload_target),$(source_dir)/$(file))
-
-src_ofld=ofldbegin.cpp ofldend.cpp
-src_ofld:=$(foreach file,$(src_ofld),$(source_dir)/$(file))
-
-headers=$(wildcard $(source_dir)/*.h) $(wildcard $(source_dir)/coi/*.h) $(wildcard $(source_dir)/orsl-lite/include/*.h)
-ifneq ($(omp_header_dir), )
-  headers+=$(imported_dir)/omp.h
-endif
-
-# Objects
-obj_liboffload_host=$(notdir $(src_liboffload_host))
-obj_liboffload_host:=$(obj_liboffload_host:.cpp=.o)
-obj_liboffload_host:=$(obj_liboffload_host:.c=.o)
-obj_liboffload_host:=$(foreach file,$(obj_liboffload_host),$(obj_host_dir)/$(file))
-
-obj_liboffload_target=$(notdir $(src_liboffload_target))
-obj_liboffload_target:=$(obj_liboffload_target:.cpp=.o)
-obj_liboffload_target:=$(obj_liboffload_target:.c=.o)
-obj_liboffload_target:=$(foreach file,$(obj_liboffload_target),$(obj_target_dir)/$(file))
-
-obj_ofld=$(notdir $(src_ofld))
-obj_ofld:=$(obj_ofld:.cpp=.o)
-obj_ofld_host=$(foreach file,$(obj_ofld),$(build_host_dir)/$(file))
-obj_ofld_target=$(foreach file,$(obj_ofld),$(build_target_dir)/$(file))
-
-# Options
-opts_common=-O2 -w -fpic -c -DCOI_LIBRARY_VERSION=2 -DMYO_SUPPORT -DOFFLOAD_DEBUG=1 -DSEP_SUPPORT -DTIMING_SUPPORT -I$(coi_include) -I$(myo_dir)/include -I$(source_dir)
-ifneq ($(omp_header_dir), )
-  opts_common+=-I$(imported_dir)
-endif
-
-opts_liboffload=-shared -Wl,-soname,liboffload.so.5 -ldl -lstdc++ -liomp5
-
-opts_liboffload_host=$(opts_liboffload) -L$(coi_lib_host) -lcoi_host -L$(myo_dir)/lib -lmyo-client
-ifneq ($(libiomp_host_dir), )
-  opts_liboffload_host+=-L$(libiomp_host_dir)
-endif
-
-opts_liboffload_target=$(opts_liboffload) -L$(coi_lib_device) -lcoi_device -L$(myo_dir)/lib -lmyo-service
-ifneq ($(libiomp_target_dir), )
-  opts_liboffload_target+=-L$(libiomp_target_dir)
-endif
-
-options_host?=
-opts_host=$(options_host) -DHOST_LIBRARY=1 -DMPSS_VERSION=$(mpss_version)
-ifeq ($(os_host), linux)
-  opts_host+=-DLINUX
-endif
-
-options_target?=
-opts_target=$(options_target) -DHOST_LIBRARY=0
-ifeq ($(os_target), linux)
-  opts_target+=-DLINUX
-endif
-ifeq ($(compiler_target), icc)
-  opts_target+=-mmic
-endif
-
-# Make targets
-.PHONY: all clean info
-
-all: info $(build_host_dir)/liboffload.so $(build_target_dir)/liboffload.so $(obj_ofld_host) $(obj_ofld_target)
-
-
-$(build_host_dir)/liboffload.so: $(build_host_dir)/liboffload.so.5 | $(build_host_dir)
-	ln -f $< $@
-
-$(build_host_dir)/liboffload.so.5: $(obj_liboffload_host) | $(build_host_dir)
-	$(compiler_host) $(opts_liboffload_host) $(opts_host) $^ -o $@
-
-$(obj_host_dir)/%.o: $(source_dir)/%.c $(headers) | $(obj_host_dir)
-	$(compiler_host) $(opts_common) $(opts_host) $< -o $@
-
-$(obj_host_dir)/%.o: $(source_dir)/%.cpp $(headers) | $(obj_host_dir)
-	$(compiler_host) $(opts_common) $(opts_host) $< -o $@
-
-$(obj_host_dir)/%.o: $(source_dir)/coi/%.cpp $(headers) | $(obj_host_dir)
-	$(compiler_host) $(opts_common) $(opts_host) $< -o $@
-
-$(obj_host_dir)/%.o: $(source_dir)/orsl-lite/lib/%.c $(headers) | $(obj_host_dir)
-	$(compiler_host) $(opts_common) $(opts_host) $< -o $@
-
-
-$(build_target_dir)/liboffload.so: $(build_target_dir)/liboffload.so.5 | $(build_target_dir)
-	ln -f $< $@
-
-$(build_target_dir)/liboffload.so.5: $(obj_liboffload_target) | $(build_target_dir)
-	$(compiler_target) $(opts_liboffload_target) $(opts_target) $^ -o $@
-
-$(obj_target_dir)/%.o: $(source_dir)/%.c $(headers) | $(obj_target_dir)
-	$(compiler_target) $(opts_common) $(opts_target) $< -o $@
-
-$(obj_target_dir)/%.o: $(source_dir)/%.cpp $(headers) | $(obj_target_dir)
-	$(compiler_target) $(opts_common) $(opts_target) $< -o $@
-
-$(obj_target_dir)/%.o: $(source_dir)/coi/%.cpp $(headers) | $(obj_target_dir)
-	$(compiler_target) $(opts_common) $(opts_target) $< -o $@
-
-$(obj_target_dir)/%.o: $(source_dir)/orsl-lite/lib/%.c $(headers) | $(obj_target_dir)
-	$(compiler_target) $(opts_common) $(opts_target) $< -o $@
-
-
-$(build_host_dir)/%.o: $(source_dir)/%.cpp $(headers) | $(build_host_dir)
-	$(compiler_host) $(opts_common) $(opts_host) $< -o $@
-
-$(build_target_dir)/%.o: $(source_dir)/%.cpp $(headers) | $(build_target_dir)
-	$(compiler_target) $(opts_common) $(opts_target) $< -o $@
-
-
-$(imported_dir)/omp.h: $(omp_header_dir)/omp.h | $(imported_dir)
-	cp $< $@
-
-
-$(build_host_dir) $(build_target_dir) $(obj_host_dir) $(obj_target_dir): | $(build_dir)
-	$(shell mkdir -p $@ >/dev/null 2>/dev/null)
-	@echo "Created $@ directory"
-
-$(build_dir):
-	$(shell mkdir -p $@ >/dev/null 2>/dev/null)
-	@echo "Created $@ directory"
-
-$(imported_dir):
-	$(shell mkdir -p $@ >/dev/null 2>/dev/null)
-	@echo "Created $@ directory"
-
-
-clean:
-	$(shell rm -rf $(build_dir))
-	@echo "Remove $(build_dir) directory"
-
-
-info:
-	@echo "root_dir = $(root_dir)"
-	@echo "build_dir = $(build_dir)"
-	@echo "mpss_dir = $(mpss_dir)"
-	@echo "mpss_version = $(mpss_version)"
-	@echo "libiomp_host_dir = $(libiomp_host_dir)"
-	@echo "libiomp_target_dir = $(libiomp_target_dir)"
-	@echo "omp_header_dir = $(omp_header_dir)"
-	@echo "os_host = $(os_host)"
-	@echo "os_target = $(os_target)"
-	@echo "compiler_host = $(compiler_host)"
-	@echo "compiler_target = $(compiler_target)"
-	@echo "options_host = $(options_host)"
-	@echo "options_target = $(options_target)"
-

offload/README.txt

@@ -1,129 +0,0 @@
-
-	       README for Intel(R) Offload Runtime Library
-	       ===========================================
-
-How to Build Documentation
-==========================
-
-The main documentation is in Doxygen* format, and this distribution
-should come with pre-built PDF documentation in doc/Reference.pdf.
-However, an HTML version can be built by executing:
-
-% doxygen doc/doxygen/config
-
-in this directory.
-
-That will produce HTML documentation in the doc/doxygen/generated
-directory, which can be accessed by pointing a web browser at the
-index.html file there.
-
-If you don't have Doxygen installed, you can download it from
-www.doxygen.org.
-
-
-Software Requirements
-=====================
-
-Intel(R) Offload Runtime Library requires additional software:
-
-1) Intel(R) OpenMP* Runtime Library.  You can either download the source
-code for that (from openmprtl.org or openmp.llvm.org) or simply use the
-compiled version distributed with the Intel compilers.
-2) Intel(R) COI Runtime Library and Intel(R) MYO Runtime Library.  These
-libraries are part of Intel(R) Manycore Platform Software Stack (MPSS).  You
-can download MPSS source code or binaries from
-software.intel.com/en-us/articles/intel-manycore-platform-software-stack-mpss.
-Binaries include host libraries for Intel(R) 64 Architecture and target
-libraries for Intel(R) Many Integrated Core Architecture.
-
-Also you will require all of the libraries that enable the target code to run
-on device.  If you target the Intel(R) Xeon Phi (TM) coprocessor, these
-libraries can be taken from MPSS too.
-
-
-How to Build the Intel(R) Offload Runtime Library
-=================================================
-
-The Makefile at the top-level will attempt to detect what it needs to
-build the Intel(R) Offload Runtime Library.  To see the default settings,
-type:
-
-make info
-
-You can change the Makefile's behavior with the following options:
-
-root_dir:	      The path to the top-level directory containing the
-		      top-level Makefile.  By default, this will take on the
-		      value of the current working directory.
-
-build_dir:	      The path to the build directory.  By default, this will
-		      take on value [root_dir]/build.
-
-mpss_dir:	      The path to the Intel(R) Manycore Platform Software
-		      Stack install directory.  By default, this will take on
-		      the value of operating system's root directory.
-
-libiomp_host_dir:     The path to the host Intel(R) OpenMP* Runtime Library.
-		      This option is required when the host compiler is other
-		      than icc.
-
-libiomp_target_dir:   The path to the target Intel(R) OpenMP* Runtime
-		      Library.  This option is required when the target
-		      compiler is other than icc.
-
-omp_header_dir:       The path to the header file <omp.h> of Intel(R) OpenMP*
-		      Runtime Library.  This option is required if either host
-		      or target compiler is other than icc.
-
-os_host:	      Operating system on host.  Currently supports only
-		      "linux" which is set by default.
-
-os_target:	      Operating system on target device.  Currently supports
-		      only "linux" which is set by default.
-
-compiler_host:	      Which compiler to use for the build of the host part.
-		      Defaults to "gcc"*.  Also supports "icc" and "clang"*.
-		      You should provide the full path to the compiler or it
-		      should be in the user's path.
-
-compiler_host:	      Which compiler to use for the build of the target part.
-		      Defaults to "gcc"*.  Also supports "icc" and "clang"*.
-		      You should provide the full path to the compiler or it
-		      should be in the user's path.
-
-options_host:	      Additional options for the host compiler.
-
-options_target:       Additional options for the target compiler.
-
-To use any of the options above, simple add <option_name>=<value>.  For
-example, if you want to build with icc instead of gcc, type:
-
-make compiler_host=icc compiler_target=icc
-
-
-Supported RTL Build Configurations
-==================================
-
-Supported Architectures: Intel(R) 64, and Intel(R) Many Integrated
-Core Architecture
-
-	      ---------------------------------------------
-	      |   icc/icl     |    gcc      |    clang    |
---------------|---------------|---------------------------|
-| Linux* OS   |      Yes      |     Yes(1)  |     Yes(1)  |
-| OS X*       |       No      |      No     |      No     |
-| Windows* OS |       No      |      No     |      No     |
------------------------------------------------------------
-
-(1) Liboffload requires _rdtsc intrinsic, which may be unsupported by some
-    versions of compiler.  In this case you need to include src/rdtsc.h
-    manually by using Makefile options options_host and options_target:
-
-    make options_host="-include src/rdtsc.h" options_target="-include src/rdtsc.h"
-
------------------------------------------------------------------------
-
-Notices
-=======
-
-*Other names and brands may be claimed as the property of others.

offload/doc/doxygen/header.tex

@@ -1,90 +0,0 @@
-% Latex header for doxygen 1.8.3.1
-\documentclass{book}
-\usepackage[a4paper,top=2.5cm,bottom=2.5cm,left=2.5cm,right=2.5cm]{geometry}
-\usepackage{makeidx}
-\usepackage{natbib}
-\usepackage{graphicx}
-\usepackage{multicol}
-\usepackage{float}
-\usepackage{listings}
-\usepackage{color}
-\usepackage{ifthen}
-\usepackage[table]{xcolor}
-\usepackage{textcomp}
-\usepackage{alltt}
-\usepackage{ifpdf}
-\ifpdf
-\usepackage[pdftex,
-            pagebackref=true,
-            colorlinks=true,
-            linkcolor=blue,
-            unicode
-           ]{hyperref}
-\else
-\usepackage[ps2pdf,
-            pagebackref=true,
-            colorlinks=true,
-            linkcolor=blue,
-            unicode
-           ]{hyperref}
-\usepackage{pspicture}
-\fi
-\usepackage[utf8]{inputenc}
-\usepackage{mathptmx}
-\usepackage[scaled=.90]{helvet}
-\usepackage{courier}
-\usepackage{sectsty}
-\usepackage{amssymb}
-\usepackage[titles]{tocloft}
-\usepackage{doxygen}
-\usepackage{fancyhdr}
-\pagestyle{fancy}
-\lstset{language=C++,inputencoding=utf8,basicstyle=\footnotesize,breaklines=true,breakatwhitespace=true,tabsize=4,numbers=left }
-\makeindex
-\setcounter{tocdepth}{3}
-\renewcommand{\footrulewidth}{0.4pt}
-\renewcommand{\familydefault}{\sfdefault}
-\hfuzz=15pt
-\setlength{\emergencystretch}{15pt}
-\hbadness=750
-\tolerance=750
-\begin{document}
-\hypersetup{pageanchor=false,citecolor=blue}
-\begin{titlepage}
-\vspace*{7cm}
-\begin{center}
-{\Large Intel\textsuperscript{\textregistered} Offload Runtime Library }\\
-\vspace*{1cm}
-{\large Generated by Doxygen $doxygenversion }\\
-\vspace*{0.5cm}
-{\small $datetime }\\
-\end{center}
-\end{titlepage}
-
-{\bf FTC Optimization Notice}
-
-Intel's compilers may or may not optimize to the same degree for non-Intel microprocessors for
-optimizations that are not unique to Intel microprocessors. These optimizations include SSE2,
-SSE3, and SSSE3 instruction sets and other optimizations. Intel does not guarantee the
-availability, functionality, or effectiveness of any optimization on microprocessors not
-manufactured by Intel.
-
-Microprocessor-dependent optimizations in this product are intended for use with Intel
-microprocessors. Certain optimizations not specific to Intel microarchitecture are reserved for
-Intel microprocessors. Please refer to the applicable product User and Reference Guides for
-more information regarding the specific instruction sets covered by this notice.
-
-Notice revision \#20110804
-
-\vspace*{0.5cm}
-
-{\bf Trademarks}
-
-Intel, Xeon, and Intel Xeon Phi are trademarks of Intel Corporation in the U.S. and/or other countries.
-
-This document is Copyright \textcopyright 2014, Intel Corporation. All rights reserved. 
-
-\pagenumbering{roman}
-\tableofcontents
-\pagenumbering{arabic}
-\hypersetup{pageanchor=true,citecolor=blue}

offload/src/cean_util.cpp

@@ -1,344 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "cean_util.h"
-#include "offload_common.h"
-
-// 1. allocate element of CeanReadRanges type
-// 2. initialized it for reading consequently contiguous ranges
-//    described by "ap" argument
-CeanReadRanges * init_read_ranges_arr_desc(const arr_desc *ap)
-{
-    CeanReadRanges * res;
-
-    // find the max contiguous range
-    int64_t rank = ap->rank - 1;
-    int64_t length = ap->dim[rank].size;
-    for (; rank >= 0; rank--) {
-        if (ap->dim[rank].stride == 1) {
-            length *= (ap->dim[rank].upper - ap->dim[rank].lower + 1);
-            if (rank > 0 && length != ap->dim[rank - 1].size) {
-                break;
-            }
-        }
-        else {
-            break;
-        }
-    }
-
-    res =(CeanReadRanges *)malloc(sizeof(CeanReadRanges) +
-                                  (ap->rank - rank) * sizeof(CeanReadDim));
-    res->current_number = 0;
-    res->range_size = length;
-    res->last_noncont_ind = rank;
-
-    // calculate number of contiguous ranges inside noncontiguous dimensions
-    int count = 1;
-    bool prev_is_cont = true;
-    int64_t offset = 0;
-
-    for (; rank >= 0; rank--) {
-        res->Dim[rank].count = count;
-        res->Dim[rank].size = ap->dim[rank].stride * ap->dim[rank].size;
-        count *= (prev_is_cont && ap->dim[rank].stride == 1? 1 :
-            (ap->dim[rank].upper - ap->dim[rank].lower +
-            ap->dim[rank].stride) / ap->dim[rank].stride);
-        prev_is_cont = false;
-        offset +=(ap->dim[rank].lower - ap->dim[rank].lindex) *
-                 ap->dim[rank].size;
-    }
-    res->range_max_number = count;
-    res -> ptr = (void*)ap->base;
-    res -> init_offset = offset;
-    return res;
-}
-
-// check if ranges described by 1 argument could be transferred into ranges
-// described by 2-nd one
-bool cean_ranges_match(
-    CeanReadRanges * read_rng1,
-    CeanReadRanges * read_rng2
-)
-{
-    return ( read_rng1 == NULL || read_rng2 == NULL ||
-            (read_rng1->range_size % read_rng2->range_size == 0 ||
-            read_rng2->range_size % read_rng1->range_size == 0));
-}
-
-// Set next offset and length and returns true for next range.
-// Returns false if the ranges are over.
-bool get_next_range(
-    CeanReadRanges * read_rng,
-    int64_t *offset
-)
-{
-    if (++read_rng->current_number > read_rng->range_max_number) {
-        read_rng->current_number = 0;
-        return false;
-    }
-    int rank = 0;
-    int num = read_rng->current_number - 1;
-    int64_t cur_offset = 0;
-    int num_loc;
-    for (; rank <= read_rng->last_noncont_ind; rank++) {
-        num_loc = num / read_rng->Dim[rank].count;
-        cur_offset += num_loc * read_rng->Dim[rank].size;
-        num = num % read_rng->Dim[rank].count;
-    }
-    *offset = cur_offset + read_rng->init_offset;
-    return true;
-}
-
-bool is_arr_desc_contiguous(const arr_desc *ap)
-{
-    int64_t rank = ap->rank - 1;
-    int64_t length = ap->dim[rank].size;
-    for (; rank >= 0; rank--) {
-        if (ap->dim[rank].stride > 1 &&
-            ap->dim[rank].upper - ap->dim[rank].lower != 0) {
-                return false;
-        }
-        else if (length != ap->dim[rank].size) {
-            for (; rank >= 0; rank--) {
-                if (ap->dim[rank].upper - ap->dim[rank].lower != 0) {
-                    return false;
-                }
-            }
-            return true;
-        }
-        length *= (ap->dim[rank].upper - ap->dim[rank].lower + 1);
-    }
-    return true;
-}
-
-int64_t cean_get_transf_size(CeanReadRanges * read_rng)
-{
-    return(read_rng->range_max_number * read_rng->range_size);
-}
-
-static uint64_t last_left, last_right;
-typedef void (*fpp)(const char *spaces, uint64_t low, uint64_t high, int esize);
-
-static void generate_one_range(
-    const char *spaces,
-    uint64_t lrange,
-    uint64_t rrange,
-    fpp fp,
-    int esize
-)
-{
-    OFFLOAD_TRACE(3,
-        "%s    generate_one_range(lrange=%p, rrange=%p, esize=%d)\n",
-        spaces, (void*)lrange, (void*)rrange, esize);
-    if (last_left == -1) {
-        // First range
-        last_left = lrange;
-    }
-    else {
-        if (lrange == last_right+1) {
-            // Extend previous range, don't print
-        }
-        else {
-            (*fp)(spaces, last_left, last_right, esize);
-            last_left = lrange;
-        }
-    }
-    last_right = rrange;
-}
-
-static void generate_mem_ranges_one_rank(
-    const char *spaces,
-    uint64_t base,
-    uint64_t rank,
-    const struct dim_desc *ddp,
-    fpp fp,
-    int esize
-)
-{
-    uint64_t lindex = ddp->lindex;
-    uint64_t lower = ddp->lower;
-    uint64_t upper = ddp->upper;
-    uint64_t stride = ddp->stride;
-    uint64_t size = ddp->size;
-    OFFLOAD_TRACE(3,
-        "%s    "
-        "generate_mem_ranges_one_rank(base=%p, rank=%lld, lindex=%lld, "
-        "lower=%lld, upper=%lld, stride=%lld, size=%lld, esize=%d)\n",
-        spaces, (void*)base, rank, lindex, lower, upper, stride, size, esize);
-    if (rank == 1) {
-        uint64_t lrange, rrange;
-        if (stride == 1) {
-            lrange = base + (lower-lindex)*size;
-            rrange = lrange + (upper-lower+1)*size - 1;
-            generate_one_range(spaces, lrange, rrange, fp, esize);
-        }
-        else {
-            for (int i=lower-lindex; i<=upper-lindex; i+=stride) {
-                lrange = base + i*size;
-                rrange = lrange + size - 1;
-                generate_one_range(spaces, lrange, rrange, fp, esize);
-            }
-        }
-    }
-    else {
-        for (int i=lower-lindex; i<=upper-lindex; i+=stride) {
-            generate_mem_ranges_one_rank(
-                spaces, base+i*size, rank-1, ddp+1, fp, esize);
-
-        }
-    }
-}
-
-static void generate_mem_ranges(
-    const char *spaces,
-    const arr_desc *adp,
-    bool deref,
-    fpp fp
-)
-{
-    uint64_t esize;
-
-    OFFLOAD_TRACE(3,
-        "%s    "
-        "generate_mem_ranges(adp=%p, deref=%d, fp)\n",
-        spaces, adp, deref);
-    last_left = -1;
-    last_right = -2;
-
-    // Element size is derived from last dimension
-    esize = adp->dim[adp->rank-1].size;
-
-    generate_mem_ranges_one_rank(
-        // For c_cean_var the base addr is the address of the data
-        // For c_cean_var_ptr the base addr is dereferenced to get to the data
-        spaces, deref ? *((uint64_t*)(adp->base)) : adp->base,
-        adp->rank, &adp->dim[0], fp, esize);
-    (*fp)(spaces, last_left, last_right, esize);
-}
-
-// returns offset and length of the data to be transferred
-void __arr_data_offset_and_length(
-    const arr_desc *adp,
-    int64_t &offset,
-    int64_t &length
-)
-{
-    int64_t rank = adp->rank - 1;
-    int64_t size = adp->dim[rank].size;
-    int64_t r_off = 0; // offset from right boundary
-
-    // find the rightmost dimension which takes just part of its
-    // range. We define it if the size of left rank is not equal
-    // the range's length between upper and lower boungaries
-    while (rank > 0) {
-        size *= (adp->dim[rank].upper - adp->dim[rank].lower + 1);
-        if (size != adp->dim[rank - 1].size) {
-            break;
-        }
-        rank--;
-    }
-
-    offset = (adp->dim[rank].lower - adp->dim[rank].lindex) *
-             adp->dim[rank].size;
-
-    // find gaps both from the left - offset and from the right - r_off
-    for (rank--; rank >= 0; rank--) {
-        offset += (adp->dim[rank].lower - adp->dim[rank].lindex) *
-                  adp->dim[rank].size;
-        r_off += adp->dim[rank].size -
-                 (adp->dim[rank + 1].upper - adp->dim[rank + 1].lindex + 1) *
-                 adp->dim[rank + 1].size;
-    }
-    length = (adp->dim[0].upper - adp->dim[0].lindex + 1) *
-             adp->dim[0].size - offset - r_off;
-}
-
-#if OFFLOAD_DEBUG > 0
-
-void print_range(
-    const char *spaces,
-    uint64_t low,
-    uint64_t high,
-    int esize
-)
-{
-    char buffer[1024];
-    char number[32];
-
-    OFFLOAD_TRACE(3, "%s        print_range(low=%p, high=%p, esize=%d)\n",
-        spaces, (void*)low, (void*)high, esize);
-
-    if (console_enabled < 4) {
-        return;
-    }
-    OFFLOAD_TRACE(4, "%s            values:\n", spaces);
-    int count = 0;
-    buffer[0] = '\0';
-    while (low <= high)
-    {
-        switch (esize)
-        {
-        case 1:
-            sprintf(number, "%d ", *((char *)low));
-            low += 1;
-            break;
-        case 2:
-            sprintf(number, "%d ", *((short *)low));
-            low += 2;
-            break;
-        case 4:
-            sprintf(number, "%d ", *((int *)low));
-            low += 4;
-            break;
-        default:
-            sprintf(number, "0x%016x ", *((uint64_t *)low));
-            low += 8;
-            break;
-        }
-        strcat(buffer, number);
-        count++;
-        if (count == 10) {
-            OFFLOAD_TRACE(4, "%s            %s\n", spaces, buffer);
-            count = 0;
-            buffer[0] = '\0';
-        }
-    }
-    if (count != 0) {
-        OFFLOAD_TRACE(4, "%s            %s\n", spaces, buffer);
-    }
-}
-
-void __arr_desc_dump(
-    const char *spaces,
-    const char *name,
-    const arr_desc *adp,
-    bool deref
-)
-{
-    OFFLOAD_TRACE(2, "%s%s CEAN expression %p\n", spaces, name, adp);
-
-    if (adp != 0) {
-        OFFLOAD_TRACE(2, "%s    base=%llx, rank=%lld\n",
-            spaces, adp->base, adp->rank);
-
-        for (int i = 0; i < adp->rank; i++) {
-            OFFLOAD_TRACE(2,
-                          "%s    dimension %d: size=%lld, lindex=%lld, "
-                          "lower=%lld, upper=%lld, stride=%lld\n",
-                          spaces, i, adp->dim[i].size, adp->dim[i].lindex,
-                          adp->dim[i].lower, adp->dim[i].upper,
-                          adp->dim[i].stride);
-        }
-        // For c_cean_var the base addr is the address of the data
-        // For c_cean_var_ptr the base addr is dereferenced to get to the data
-        generate_mem_ranges(spaces, adp, deref, &print_range);
-    }
-}
-#endif // OFFLOAD_DEBUG

offload/src/cean_util.h

@@ -1,101 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef CEAN_UTIL_H_INCLUDED
-#define CEAN_UTIL_H_INCLUDED
-
-#if MPSS_VERSION > 33
-#include <source/COIBuffer_source.h>
-#endif
-#include <stdint.h>
-
-#if MPSS_VERSION <= 33
-// CEAN expression representation
-struct dim_desc {
-    int64_t size;       // Length of data type
-    int64_t lindex;     // Lower index
-    int64_t lower;      // Lower section bound
-    int64_t upper;      // Upper section bound
-    int64_t stride;     // Stride
-};
-
-struct arr_desc {
-    int64_t base;       // Base address
-    int64_t rank;       // Rank of array
-    dim_desc dim[1];
-};
-#endif
-
-struct CeanReadDim {
-    int64_t count; // The number of elements in this dimension
-    int64_t size;  // The number of bytes between successive
-                   // elements in this dimension.
-};
-
-struct CeanReadRanges {
-    void *  ptr;
-    int64_t current_number;   // the number of ranges read
-    int64_t range_max_number; // number of contiguous ranges
-    int64_t range_size;       // size of max contiguous range
-    int     last_noncont_ind; // size of Dim array
-    int64_t init_offset;      // offset of 1-st element from array left bound
-    CeanReadDim Dim[1];
-};
-
-// array descriptor length
-#define __arr_desc_length(rank) \
-    (sizeof(int64_t) + sizeof(dim_desc) * (rank))
-
-// returns offset and length of the data to be transferred
-void __arr_data_offset_and_length(const arr_desc *adp,
-                                  int64_t &offset,
-                                  int64_t &length);
-
-// define if data array described by argument is contiguous one
-bool is_arr_desc_contiguous(const arr_desc *ap);
-
-// allocate element of CeanReadRanges type initialized
-// to read consequently contiguous ranges described by "ap" argument
-CeanReadRanges * init_read_ranges_arr_desc(const arr_desc *ap);
-
-// check if ranges described by 1 argument could be transferred into ranges
-// described by 2-nd one
-bool cean_ranges_match(
-    CeanReadRanges * read_rng1,
-    CeanReadRanges * read_rng2
-);
-
-// first argument - returned value by call to init_read_ranges_arr_desc.
-// returns true if offset and length of next range is set successfuly.
-// returns false if the ranges is over.
-bool get_next_range(
-    CeanReadRanges * read_rng,
-    int64_t *offset
-);
-
-// returns number of transferred bytes
-int64_t cean_get_transf_size(CeanReadRanges * read_rng);
-
-#if OFFLOAD_DEBUG > 0
-// prints array descriptor contents to stderr
-void    __arr_desc_dump(
-    const char *spaces,
-    const char *name,
-    const arr_desc *adp,
-    bool dereference);
-#else
-#define __arr_desc_dump(
-    spaces,
-    name,
-    adp,
-    dereference)
-#endif // OFFLOAD_DEBUG
-
-#endif // CEAN_UTIL_H_INCLUDED

offload/src/coi/coi_client.cpp

@@ -1,350 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-// The COI host interface
-
-#include "coi_client.h"
-#include "../offload_common.h"
-
-namespace COI {
-
-#define COI_VERSION1    "COI_1.0"
-#define COI_VERSION2    "COI_2.0"
-
-bool            is_available;
-static void*    lib_handle;
-
-// pointers to functions from COI library
-COIRESULT (*EngineGetCount)(COI_ISA_TYPE, uint32_t*);
-COIRESULT (*EngineGetHandle)(COI_ISA_TYPE, uint32_t, COIENGINE*);
-
-COIRESULT (*ProcessCreateFromMemory)(COIENGINE, const char*, const void*,
-                                     uint64_t, int, const char**, uint8_t,
-                                     const char**, uint8_t, const char*,
-                                     uint64_t, const char*, const char*,
-                                     uint64_t, COIPROCESS*);
-COIRESULT (*ProcessDestroy)(COIPROCESS, int32_t, uint8_t, int8_t*, uint32_t*);
-COIRESULT (*ProcessGetFunctionHandles)(COIPROCESS, uint32_t, const char**,
-                                       COIFUNCTION*);
-COIRESULT (*ProcessLoadLibraryFromMemory)(COIPROCESS, const void*, uint64_t,
-                                          const char*, const char*,
-                                          const char*, uint64_t, uint32_t,
-                                          COILIBRARY*);
-COIRESULT (*ProcessRegisterLibraries)(uint32_t, const void**, const uint64_t*,
-                                      const char**, const uint64_t*);
-
-COIRESULT (*PipelineCreate)(COIPROCESS, COI_CPU_MASK, uint32_t, COIPIPELINE*);
-COIRESULT (*PipelineDestroy)(COIPIPELINE);
-COIRESULT (*PipelineRunFunction)(COIPIPELINE, COIFUNCTION, uint32_t,
-                                 const COIBUFFER*, const COI_ACCESS_FLAGS*,
-                                 uint32_t, const COIEVENT*, const void*,
-                                 uint16_t, void*, uint16_t, COIEVENT*);
-
-COIRESULT (*BufferCreate)(uint64_t, COI_BUFFER_TYPE, uint32_t, const void*,
-                          uint32_t, const COIPROCESS*, COIBUFFER*);
-COIRESULT (*BufferCreateFromMemory)(uint64_t, COI_BUFFER_TYPE, uint32_t,
-                                    void*, uint32_t, const COIPROCESS*,
-                                    COIBUFFER*);
-COIRESULT (*BufferDestroy)(COIBUFFER);
-COIRESULT (*BufferMap)(COIBUFFER, uint64_t, uint64_t, COI_MAP_TYPE, uint32_t,
-                       const COIEVENT*, COIEVENT*, COIMAPINSTANCE*, void**);
-COIRESULT (*BufferUnmap)(COIMAPINSTANCE, uint32_t, const COIEVENT*, COIEVENT*);
-COIRESULT (*BufferWrite)(COIBUFFER, uint64_t, const void*, uint64_t,
-                         COI_COPY_TYPE, uint32_t, const COIEVENT*, COIEVENT*);
-COIRESULT (*BufferRead)(COIBUFFER, uint64_t, void*, uint64_t, COI_COPY_TYPE,
-                        uint32_t, const COIEVENT*, COIEVENT*);
-COIRESULT (*BufferCopy)(COIBUFFER, COIBUFFER, uint64_t, uint64_t, uint64_t,
-                        COI_COPY_TYPE, uint32_t, const COIEVENT*, COIEVENT*);
-COIRESULT (*BufferGetSinkAddress)(COIBUFFER, uint64_t*);
-COIRESULT (*BufferSetState)(COIBUFFER, COIPROCESS, COI_BUFFER_STATE,
-                            COI_BUFFER_MOVE_FLAG, uint32_t,
-                            const   COIEVENT*, COIEVENT*);
-
-COIRESULT (*EventWait)(uint16_t, const COIEVENT*, int32_t, uint8_t, uint32_t*,
-                       uint32_t*);
-
-uint64_t  (*PerfGetCycleFrequency)(void);
-
-bool init(void)
-{
-#ifndef TARGET_WINNT
-    const char *lib_name = "libcoi_host.so.0";
-#else // TARGET_WINNT
-    const char *lib_name = "coi_host.dll";
-#endif // TARGET_WINNT
-
-    OFFLOAD_DEBUG_TRACE(2, "Loading COI library %s ...\n", lib_name);
-    lib_handle = DL_open(lib_name);
-    if (lib_handle == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to load the library\n");
-        return false;
-    }
-
-    EngineGetCount =
-        (COIRESULT (*)(COI_ISA_TYPE, uint32_t*))
-            DL_sym(lib_handle, "COIEngineGetCount", COI_VERSION1);
-    if (EngineGetCount == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIEngineGetCount");
-        fini();
-        return false;
-    }
-
-    EngineGetHandle =
-        (COIRESULT (*)(COI_ISA_TYPE, uint32_t, COIENGINE*))
-            DL_sym(lib_handle, "COIEngineGetHandle", COI_VERSION1);
-    if (EngineGetHandle == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIEngineGetHandle");
-        fini();
-        return false;
-    }
-
-    ProcessCreateFromMemory =
-        (COIRESULT (*)(COIENGINE, const char*, const void*, uint64_t, int,
-                       const char**, uint8_t, const char**, uint8_t,
-                       const char*, uint64_t, const char*, const char*,
-                       uint64_t, COIPROCESS*))
-            DL_sym(lib_handle, "COIProcessCreateFromMemory", COI_VERSION1);
-    if (ProcessCreateFromMemory == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIProcessCreateFromMemory");
-        fini();
-        return false;
-    }
-
-    ProcessDestroy =
-        (COIRESULT (*)(COIPROCESS, int32_t, uint8_t, int8_t*,
-                       uint32_t*))
-            DL_sym(lib_handle, "COIProcessDestroy", COI_VERSION1);
-    if (ProcessDestroy == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIProcessDestroy");
-        fini();
-        return false;
-    }
-
-    ProcessGetFunctionHandles =
-        (COIRESULT (*)(COIPROCESS, uint32_t, const char**, COIFUNCTION*))
-            DL_sym(lib_handle, "COIProcessGetFunctionHandles", COI_VERSION1);
-    if (ProcessGetFunctionHandles == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIProcessGetFunctionHandles");
-        fini();
-        return false;
-    }
-
-    ProcessLoadLibraryFromMemory =
-        (COIRESULT (*)(COIPROCESS, const void*, uint64_t, const char*,
-                       const char*, const char*, uint64_t, uint32_t,
-                       COILIBRARY*))
-            DL_sym(lib_handle, "COIProcessLoadLibraryFromMemory", COI_VERSION2);
-    if (ProcessLoadLibraryFromMemory == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIProcessLoadLibraryFromMemory");
-        fini();
-        return false;
-    }
-
-    ProcessRegisterLibraries =
-        (COIRESULT (*)(uint32_t, const void**, const uint64_t*, const char**,
-                       const uint64_t*))
-            DL_sym(lib_handle, "COIProcessRegisterLibraries", COI_VERSION1);
-    if (ProcessRegisterLibraries == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIProcessRegisterLibraries");
-        fini();
-        return false;
-    }
-
-    PipelineCreate =
-        (COIRESULT (*)(COIPROCESS, COI_CPU_MASK, uint32_t, COIPIPELINE*))
-            DL_sym(lib_handle, "COIPipelineCreate", COI_VERSION1);
-    if (PipelineCreate == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIPipelineCreate");
-        fini();
-        return false;
-    }
-
-    PipelineDestroy =
-        (COIRESULT (*)(COIPIPELINE))
-            DL_sym(lib_handle, "COIPipelineDestroy", COI_VERSION1);
-    if (PipelineDestroy == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIPipelineDestroy");
-        fini();
-        return false;
-    }
-
-    PipelineRunFunction =
-        (COIRESULT (*)(COIPIPELINE, COIFUNCTION, uint32_t, const COIBUFFER*,
-                       const COI_ACCESS_FLAGS*, uint32_t, const COIEVENT*,
-                       const void*, uint16_t, void*, uint16_t, COIEVENT*))
-            DL_sym(lib_handle, "COIPipelineRunFunction", COI_VERSION1);
-    if (PipelineRunFunction == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIPipelineRunFunction");
-        fini();
-        return false;
-    }
-
-    BufferCreate =
-        (COIRESULT (*)(uint64_t, COI_BUFFER_TYPE, uint32_t, const void*,
-                       uint32_t, const COIPROCESS*, COIBUFFER*))
-            DL_sym(lib_handle, "COIBufferCreate", COI_VERSION1);
-    if (BufferCreate == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferCreate");
-        fini();
-        return false;
-    }
-
-    BufferCreateFromMemory =
-        (COIRESULT (*)(uint64_t, COI_BUFFER_TYPE, uint32_t, void*,
-                       uint32_t, const COIPROCESS*, COIBUFFER*))
-            DL_sym(lib_handle, "COIBufferCreateFromMemory", COI_VERSION1);
-    if (BufferCreateFromMemory == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferCreateFromMemory");
-        fini();
-        return false;
-    }
-
-    BufferDestroy =
-        (COIRESULT (*)(COIBUFFER))
-            DL_sym(lib_handle, "COIBufferDestroy", COI_VERSION1);
-    if (BufferDestroy == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferDestroy");
-        fini();
-        return false;
-    }
-
-    BufferMap =
-        (COIRESULT (*)(COIBUFFER, uint64_t, uint64_t, COI_MAP_TYPE, uint32_t,
-                       const COIEVENT*, COIEVENT*, COIMAPINSTANCE*,
-                       void**))
-            DL_sym(lib_handle, "COIBufferMap", COI_VERSION1);
-    if (BufferMap == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferMap");
-        fini();
-        return false;
-    }
-
-    BufferUnmap =
-        (COIRESULT (*)(COIMAPINSTANCE, uint32_t, const COIEVENT*,
-                       COIEVENT*))
-            DL_sym(lib_handle, "COIBufferUnmap", COI_VERSION1);
-    if (BufferUnmap == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferUnmap");
-        fini();
-        return false;
-    }
-
-    BufferWrite =
-        (COIRESULT (*)(COIBUFFER, uint64_t, const void*, uint64_t,
-                       COI_COPY_TYPE, uint32_t, const COIEVENT*,
-                       COIEVENT*))
-            DL_sym(lib_handle, "COIBufferWrite", COI_VERSION1);
-    if (BufferWrite == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferWrite");
-        fini();
-        return false;
-    }
-
-    BufferRead =
-        (COIRESULT (*)(COIBUFFER, uint64_t, void*, uint64_t,
-                                     COI_COPY_TYPE, uint32_t,
-                                     const COIEVENT*, COIEVENT*))
-            DL_sym(lib_handle, "COIBufferRead", COI_VERSION1);
-    if (BufferRead == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferRead");
-        fini();
-        return false;
-    }
-
-    BufferCopy =
-        (COIRESULT (*)(COIBUFFER, COIBUFFER, uint64_t, uint64_t, uint64_t,
-                       COI_COPY_TYPE, uint32_t, const COIEVENT*,
-                       COIEVENT*))
-            DL_sym(lib_handle, "COIBufferCopy", COI_VERSION1);
-    if (BufferCopy == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferCopy");
-        fini();
-        return false;
-    }
-
-    BufferGetSinkAddress =
-        (COIRESULT (*)(COIBUFFER, uint64_t*))
-            DL_sym(lib_handle, "COIBufferGetSinkAddress", COI_VERSION1);
-    if (BufferGetSinkAddress == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferGetSinkAddress");
-        fini();
-        return false;
-    }
-
-    BufferSetState =
-        (COIRESULT(*)(COIBUFFER, COIPROCESS, COI_BUFFER_STATE,
-                      COI_BUFFER_MOVE_FLAG, uint32_t, const COIEVENT*,
-                      COIEVENT*))
-            DL_sym(lib_handle, "COIBufferSetState", COI_VERSION1);
-    if (BufferSetState == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIBufferSetState");
-        fini();
-        return false;
-    }
-
-    EventWait =
-        (COIRESULT (*)(uint16_t, const COIEVENT*, int32_t, uint8_t,
-                       uint32_t*, uint32_t*))
-            DL_sym(lib_handle, "COIEventWait", COI_VERSION1);
-    if (EventWait == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIEventWait");
-        fini();
-        return false;
-    }
-
-    PerfGetCycleFrequency =
-        (uint64_t (*)(void))
-            DL_sym(lib_handle, "COIPerfGetCycleFrequency", COI_VERSION1);
-    if (PerfGetCycleFrequency == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in COI library\n",
-                            "COIPerfGetCycleFrequency");
-        fini();
-        return false;
-    }
-
-    is_available = true;
-
-    return true;
-}
-
-void fini(void)
-{
-    is_available = false;
-
-    if (lib_handle != 0) {
-#ifndef TARGET_WINNT
-        DL_close(lib_handle);
-#endif // TARGET_WINNT
-        lib_handle = 0;
-    }
-}
-
-} // namespace COI

offload/src/coi/coi_client.h

@@ -1,118 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-// The interface between offload library and the COI API on the host
-
-#ifndef COI_CLIENT_H_INCLUDED
-#define COI_CLIENT_H_INCLUDED
-
-#include <common/COIPerf_common.h>
-#include <source/COIEngine_source.h>
-#include <source/COIProcess_source.h>
-#include <source/COIPipeline_source.h>
-#include <source/COIBuffer_source.h>
-#include <source/COIEvent_source.h>
-
-#include <string.h>
-
-#include "../liboffload_error_codes.h"
-#include "../offload_util.h"
-
-#define MIC_ENGINES_MAX     128
-
-#if MIC_ENGINES_MAX < COI_MAX_ISA_MIC_DEVICES
-#error MIC_ENGINES_MAX need to be increased
-#endif
-
-// COI library interface
-namespace COI {
-
-extern bool init(void);
-extern void fini(void);
-
-extern bool is_available;
-
-// pointers to functions from COI library
-extern COIRESULT (*EngineGetCount)(COI_ISA_TYPE, uint32_t*);
-extern COIRESULT (*EngineGetHandle)(COI_ISA_TYPE, uint32_t, COIENGINE*);
-
-extern COIRESULT (*ProcessCreateFromMemory)(COIENGINE, const char*,
-                                           const void*, uint64_t, int,
-                                           const char**, uint8_t,
-                                           const char**, uint8_t,
-                                           const char*, uint64_t,
-                                           const char*,
-                                           const char*, uint64_t,
-                                           COIPROCESS*);
-extern COIRESULT (*ProcessDestroy)(COIPROCESS, int32_t, uint8_t,
-                                  int8_t*, uint32_t*);
-extern COIRESULT (*ProcessGetFunctionHandles)(COIPROCESS, uint32_t,
-                                             const char**,
-                                             COIFUNCTION*);
-extern COIRESULT (*ProcessLoadLibraryFromMemory)(COIPROCESS,
-                                                const void*,
-                                                uint64_t,
-                                                const char*,
-                                                const char*,
-                                                const char*,
-                                                uint64_t,
-                                                uint32_t,
-                                                COILIBRARY*);
-extern COIRESULT (*ProcessRegisterLibraries)(uint32_t,
-                                            const void**,
-                                            const uint64_t*,
-                                            const char**,
-                                            const uint64_t*);
-
-extern COIRESULT (*PipelineCreate)(COIPROCESS, COI_CPU_MASK, uint32_t,
-                                  COIPIPELINE*);
-extern COIRESULT (*PipelineDestroy)(COIPIPELINE);
-extern COIRESULT (*PipelineRunFunction)(COIPIPELINE, COIFUNCTION,
-                                       uint32_t, const COIBUFFER*,
-                                       const COI_ACCESS_FLAGS*,
-                                       uint32_t, const COIEVENT*,
-                                       const void*, uint16_t, void*,
-                                       uint16_t, COIEVENT*);
-
-extern COIRESULT (*BufferCreate)(uint64_t, COI_BUFFER_TYPE, uint32_t,
-                                const void*, uint32_t,
-                                const COIPROCESS*, COIBUFFER*);
-extern COIRESULT (*BufferCreateFromMemory)(uint64_t, COI_BUFFER_TYPE,
-                                          uint32_t, void*,
-                                          uint32_t, const COIPROCESS*,
-                                          COIBUFFER*);
-extern COIRESULT (*BufferDestroy)(COIBUFFER);
-extern COIRESULT (*BufferMap)(COIBUFFER, uint64_t, uint64_t,
-                             COI_MAP_TYPE, uint32_t, const COIEVENT*,
-                             COIEVENT*, COIMAPINSTANCE*, void**);
-extern COIRESULT (*BufferUnmap)(COIMAPINSTANCE, uint32_t,
-                               const COIEVENT*, COIEVENT*);
-extern COIRESULT (*BufferWrite)(COIBUFFER, uint64_t, const void*,
-                               uint64_t, COI_COPY_TYPE, uint32_t,
-                               const COIEVENT*, COIEVENT*);
-extern COIRESULT (*BufferRead)(COIBUFFER, uint64_t, void*, uint64_t,
-                              COI_COPY_TYPE, uint32_t,
-                              const COIEVENT*, COIEVENT*);
-extern COIRESULT (*BufferCopy)(COIBUFFER, COIBUFFER, uint64_t, uint64_t,
-                              uint64_t, COI_COPY_TYPE, uint32_t,
-                              const COIEVENT*, COIEVENT*);
-extern COIRESULT (*BufferGetSinkAddress)(COIBUFFER, uint64_t*);
-extern COIRESULT (*BufferSetState)(COIBUFFER, COIPROCESS, COI_BUFFER_STATE,
-                                   COI_BUFFER_MOVE_FLAG, uint32_t,
-                                   const   COIEVENT*, COIEVENT*);
-
-extern COIRESULT (*EventWait)(uint16_t, const COIEVENT*, int32_t,
-                           uint8_t, uint32_t*, uint32_t*);
-
-extern uint64_t  (*PerfGetCycleFrequency)(void);
-
-} // namespace COI
-
-#endif // COI_CLIENT_H_INCLUDED

offload/src/coi/coi_server.cpp

@@ -1,130 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-// The COI interface on the target
-
-#include "coi_server.h"
-
-#include "../offload_target.h"
-#include "../offload_timer.h"
-#ifdef MYO_SUPPORT
-#include "../offload_myo_target.h"      // for __offload_myoLibInit/Fini
-#endif // MYO_SUPPORT
-
-COINATIVELIBEXPORT
-void server_compute(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    OffloadDescriptor::offload(buffer_count, buffers,
-                               misc_data, misc_data_len,
-                               return_data, return_data_len);
-}
-
-COINATIVELIBEXPORT
-void server_init(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    struct init_data {
-        int  device_index;
-        int  devices_total;
-        int  console_level;
-        int  offload_report_level;
-    } *data = (struct init_data*) misc_data;
-
-    // set device index and number of total devices
-    mic_index = data->device_index;
-    mic_engines_total = data->devices_total;
-
-    // initialize trace level
-    console_enabled = data->console_level;
-    offload_report_level = data->offload_report_level;
-
-    // return back the process id
-    *((pid_t*) return_data) = getpid();
-}
-
-COINATIVELIBEXPORT
-void server_var_table_size(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    struct Params {
-        int64_t nelems;
-        int64_t length;
-    } *params;
-
-    params = static_cast<Params*>(return_data);
-    params->length = __offload_vars.table_size(params->nelems);
-}
-
-COINATIVELIBEXPORT
-void server_var_table_copy(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    __offload_vars.table_copy(buffers[0], *static_cast<int64_t*>(misc_data));
-}
-
-#ifdef MYO_SUPPORT
-// temporary workaround for blocking behavior of myoiLibInit/Fini calls
-COINATIVELIBEXPORT
-void server_myoinit(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    __offload_myoLibInit();
-}
-
-COINATIVELIBEXPORT
-void server_myofini(
-    uint32_t  buffer_count,
-    void**    buffers,
-    uint64_t* buffers_len,
-    void*     misc_data,
-    uint16_t  misc_data_len,
-    void*     return_data,
-    uint16_t  return_data_len
-)
-{
-    __offload_myoLibFini();
-}
-#endif // MYO_SUPPORT

offload/src/coi/coi_server.h

@@ -1,74 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-//The interface between offload library and the COI API on the target.
-
-#ifndef COI_SERVER_H_INCLUDED
-#define COI_SERVER_H_INCLUDED
-
-#include <common/COIEngine_common.h>
-#include <common/COIPerf_common.h>
-#include <sink/COIProcess_sink.h>
-#include <sink/COIPipeline_sink.h>
-#include <sink/COIBuffer_sink.h>
-#include <list>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include "../liboffload_error_codes.h"
-
-// wrappers for COI API
-#define PipelineStartExecutingRunFunctions() \
-    { \
-        COIRESULT res = COIPipelineStartExecutingRunFunctions(); \
-        if (res != COI_SUCCESS) { \
-            LIBOFFLOAD_ERROR(c_pipeline_start_run_funcs, mic_index, res); \
-            exit(1); \
-        } \
-    }
-
-#define ProcessWaitForShutdown() \
-    { \
-        COIRESULT res = COIProcessWaitForShutdown(); \
-        if (res != COI_SUCCESS) { \
-            LIBOFFLOAD_ERROR(c_process_wait_shutdown, mic_index, res); \
-            exit(1); \
-        } \
-    }
-
-#define BufferAddRef(buf) \
-    { \
-        COIRESULT res = COIBufferAddRef(buf); \
-        if (res != COI_SUCCESS) { \
-            LIBOFFLOAD_ERROR(c_buf_add_ref, mic_index, res); \
-            exit(1); \
-        } \
-    }
-
-#define BufferReleaseRef(buf) \
-    { \
-        COIRESULT res = COIBufferReleaseRef(buf); \
-        if (res != COI_SUCCESS) { \
-            LIBOFFLOAD_ERROR(c_buf_release_ref, mic_index, res); \
-            exit(1); \
-        } \
-    }
-
-#define EngineGetIndex(index) \
-    { \
-        COI_ISA_TYPE isa_type; \
-        COIRESULT res = COIEngineGetIndex(&isa_type, index); \
-        if (res != COI_SUCCESS) { \
-            LIBOFFLOAD_ERROR(c_get_engine_index, mic_index, res); \
-            exit(1); \
-        } \
-    }
-
-#endif // COI_SERVER_H_INCLUDED

offload/src/compiler_if_host.cpp

@@ -1,323 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "compiler_if_host.h"
-
-#include <malloc.h>
-#ifndef TARGET_WINNT
-#include <alloca.h>
-#endif // TARGET_WINNT
-
-// Global counter on host. 
-// This variable is used if P2OPT_offload_do_data_persistence == 2.
-// The variable used to identify offload constructs contained in one procedure.
-// Increment of OFFLOAD_CALL_COUNT is inserted at entries of HOST routines with
-// offload constructs.
-static int offload_call_count = 0;
-
-extern "C" OFFLOAD OFFLOAD_TARGET_ACQUIRE(
-    TARGET_TYPE      target_type,
-    int              target_number,
-    int              is_optional,
-    _Offload_status* status,
-    const char*      file,
-    uint64_t         line
-)
-{
-    bool retval;
-    OFFLOAD ofld;
-
-    // initialize status
-    if (status != 0) {
-        status->result = OFFLOAD_UNAVAILABLE;
-        status->device_number = -1;
-        status->data_sent = 0;
-        status->data_received = 0;
-    }
-
-    // make sure libray is initialized
-    retval = __offload_init_library();
-
-    // OFFLOAD_TIMER_INIT must follow call to __offload_init_library
-    OffloadHostTimerData * timer_data = OFFLOAD_TIMER_INIT(file, line);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_total_offload);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_initialize);
-
-    // initialize all devices is init_type is on_offload_all
-    if (retval && __offload_init_type == c_init_on_offload_all) {
-        for (int i = 0; i < mic_engines_total; i++) {
-             mic_engines[i].init();
-        }
-    }
-    OFFLOAD_TIMER_STOP(timer_data, c_offload_host_initialize);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_target_acquire);
-
-    if (target_type == TARGET_HOST) {
-        // Host always available
-        retval = true;
-    }
-    else if (target_type == TARGET_MIC) {
-        if (target_number >= -1) {
-            if (retval) {
-                if (target_number >= 0) {
-                    // User provided the device number
-                    target_number = target_number % mic_engines_total;
-                }
-                else {
-                    // use device 0
-                    target_number = 0;
-                }
-
-                // reserve device in ORSL
-                if (is_optional) {
-                    if (!ORSL::try_reserve(target_number)) {
-                        target_number = -1;
-                    }
-                }
-                else {
-                    if (!ORSL::reserve(target_number)) {
-                        target_number = -1;
-                    }
-                }
-
-                // initialize device
-                if (target_number >= 0 &&
-                    __offload_init_type == c_init_on_offload) {
-                    OFFLOAD_TIMER_START(timer_data, c_offload_host_initialize);
-                    mic_engines[target_number].init();
-                    OFFLOAD_TIMER_STOP(timer_data, c_offload_host_initialize);
-                }
-            }
-            else {
-                // fallback to CPU
-                target_number = -1;
-            }
-
-            if (target_number < 0 || !retval) {
-                if (!is_optional && status == 0) {
-                    LIBOFFLOAD_ERROR(c_device_is_not_available);
-                    exit(1);
-                }
-
-                retval = false;
-            }
-        }
-        else {
-            LIBOFFLOAD_ERROR(c_invalid_device_number);
-            exit(1);
-        }
-    }
-
-    if (retval) {
-        ofld = new OffloadDescriptor(target_number, status,
-                                     !is_optional, false, timer_data);
-        OFFLOAD_TIMER_HOST_MIC_NUM(timer_data, target_number);
-        Offload_Report_Prolog(timer_data);
-        OFFLOAD_DEBUG_TRACE_1(2, timer_data->offload_number, c_offload_start,
-                              "Starting offload: target_type = %d, "
-                              "number = %d, is_optional = %d\n",
-                              target_type, target_number, is_optional);
-
-        OFFLOAD_TIMER_STOP(timer_data, c_offload_host_target_acquire);
-    }
-    else {
-        ofld = NULL;
-
-        OFFLOAD_TIMER_STOP(timer_data, c_offload_host_target_acquire);
-        OFFLOAD_TIMER_STOP(timer_data, c_offload_host_total_offload);
-        offload_report_free_data(timer_data);
-    }
-
-    return ofld;
-}
-
-extern "C" OFFLOAD OFFLOAD_TARGET_ACQUIRE1(
-    const int*  device_num,
-    const char* file,
-    uint64_t    line
-)
-{
-    int target_number;
-
-    // make sure libray is initialized and at least one device is available
-    if (!__offload_init_library()) {
-        LIBOFFLOAD_ERROR(c_device_is_not_available);
-        exit(1);
-    }
-
-    // OFFLOAD_TIMER_INIT must follow call to __offload_init_library
-
-    OffloadHostTimerData * timer_data = OFFLOAD_TIMER_INIT(file, line);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_total_offload);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_initialize);
-
-    if (__offload_init_type == c_init_on_offload_all) {
-        for (int i = 0; i < mic_engines_total; i++) {
-             mic_engines[i].init();
-        }
-    }
-
-    OFFLOAD_TIMER_STOP(timer_data, c_offload_host_initialize);
-
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_target_acquire);
-
-    // use default device number if it is not provided
-    if (device_num != 0) {
-        target_number = *device_num;
-    }
-    else {
-        target_number = __omp_device_num;
-    }
-
-    // device number should be a non-negative integer value
-    if (target_number < 0) {
-        LIBOFFLOAD_ERROR(c_omp_invalid_device_num);
-        exit(1);
-    }
-
-    // should we do this for OpenMP?
-    target_number %= mic_engines_total;
-
-    // reserve device in ORSL
-    if (!ORSL::reserve(target_number)) {
-        LIBOFFLOAD_ERROR(c_device_is_not_available);
-        exit(1);
-    }
-
-    // initialize device(s)
-    OFFLOAD_TIMER_START(timer_data, c_offload_host_initialize);
-
-    if (__offload_init_type == c_init_on_offload) {
-        mic_engines[target_number].init();
-    }
-
-    OFFLOAD_TIMER_STOP(timer_data, c_offload_host_initialize);
-
-    OFFLOAD ofld =
-        new OffloadDescriptor(target_number, 0, true, true, timer_data);
-
-    OFFLOAD_TIMER_HOST_MIC_NUM(timer_data, target_number);
-
-    Offload_Report_Prolog(timer_data);
-
-    OFFLOAD_DEBUG_TRACE_1(2, timer_data->offload_number, c_offload_start,
-                          "Starting OpenMP offload, device = %d\n",
-                          target_number);
-
-    OFFLOAD_TIMER_STOP(timer_data, c_offload_host_target_acquire);
-
-    return ofld;
-}
-
-int offload_offload_wrap(
-    OFFLOAD ofld,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void **waits,
-    const void **signal,
-    int entry_id,
-    const void *stack_addr
-)
-{
-    bool ret = ofld->offload(name, is_empty, vars, vars2, num_vars,
-                             waits, num_waits, signal, entry_id, stack_addr);
-    if (!ret || signal == 0) {
-        delete ofld;
-    }
-    return ret;
-}
-
-extern "C" int OFFLOAD_OFFLOAD1(
-    OFFLOAD ofld,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void **waits,
-    const void **signal
-)
-{
-    return offload_offload_wrap(ofld, name, is_empty,
-                            num_vars, vars, vars2,
-                            num_waits, waits,
-                            signal, NULL, NULL);
-}
-
-extern "C" int OFFLOAD_OFFLOAD2(
-    OFFLOAD ofld,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void** waits,
-    const void** signal,
-    int entry_id,
-    const void *stack_addr
-)
-{
-    return offload_offload_wrap(ofld, name, is_empty,
-                            num_vars, vars, vars2,
-                            num_waits, waits,
-                            signal, entry_id, stack_addr);
-}
-
-extern "C" int OFFLOAD_OFFLOAD(
-    OFFLOAD ofld,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void **waits,
-    const void *signal,
-    int entry_id,
-    const void *stack_addr
-)
-{
-    // signal is passed by reference now
-    const void **signal_new = (signal != 0) ? &signal : 0;
-    const void **waits_new = 0;
-    int num_waits_new = 0;
-
-    // remove NULL values from the list of signals to wait for
-    if (num_waits > 0) {
-        waits_new = (const void**) alloca(sizeof(void*) * num_waits);
-        for (int i = 0; i < num_waits; i++) {
-            if (waits[i] != 0) {
-                waits_new[num_waits_new++] = waits[i];
-            }
-        }
-    }
-
-    return OFFLOAD_OFFLOAD1(ofld, name, is_empty,
-                            num_vars, vars, vars2,
-                            num_waits_new, waits_new,
-                            signal_new);
-}
-
-extern "C" int OFFLOAD_CALL_COUNT()
-{
-    offload_call_count++;
-    return offload_call_count;
-}

offload/src/compiler_if_host.h

@@ -1,133 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*! \file
-    \brief The interface between compiler-generated host code and runtime library
-*/
-
-#ifndef COMPILER_IF_HOST_H_INCLUDED
-#define COMPILER_IF_HOST_H_INCLUDED
-
-#include "offload_host.h"
-
-#define OFFLOAD_TARGET_ACQUIRE          OFFLOAD_PREFIX(target_acquire)
-#define OFFLOAD_TARGET_ACQUIRE1         OFFLOAD_PREFIX(target_acquire1)
-#define OFFLOAD_OFFLOAD                 OFFLOAD_PREFIX(offload)
-#define OFFLOAD_OFFLOAD1                OFFLOAD_PREFIX(offload1)
-#define OFFLOAD_OFFLOAD2                OFFLOAD_PREFIX(offload2)
-#define OFFLOAD_CALL_COUNT              OFFLOAD_PREFIX(offload_call_count)
-
-
-/*! \fn OFFLOAD_TARGET_ACQUIRE
-    \brief Attempt to acquire the target.
-    \param target_type   The type of target.
-    \param target_number The device number.
-    \param is_optional   Whether CPU fall-back is allowed.
-    \param status        Address of variable to hold offload status.
-    \param file          Filename in which this offload occurred.
-    \param line          Line number in the file where this offload occurred.
-*/
-extern "C" OFFLOAD OFFLOAD_TARGET_ACQUIRE(
-    TARGET_TYPE      target_type,
-    int              target_number,
-    int              is_optional,
-    _Offload_status* status,
-    const char*      file,
-    uint64_t         line
-);
-
-/*! \fn OFFLOAD_TARGET_ACQUIRE1
-    \brief Acquire the target for offload (OpenMP).
-    \param device_number Device number or null if not specified.
-    \param file          Filename in which this offload occurred
-    \param line          Line number in the file where this offload occurred.
-*/
-extern "C" OFFLOAD OFFLOAD_TARGET_ACQUIRE1(
-    const int*      device_number,
-    const char*     file,
-    uint64_t        line
-);
-
-/*! \fn OFFLOAD_OFFLOAD1
-    \brief Run function on target using interface for old data persistence.
-    \param o Offload descriptor created by OFFLOAD_TARGET_ACQUIRE.
-    \param name Name of offload entry point.
-    \param is_empty If no code to execute (e.g. offload_transfer)
-    \param num_vars Number of variable descriptors.
-    \param vars Pointer to VarDesc array.
-    \param vars2 Pointer to VarDesc2 array.
-    \param num_waits Number of "wait" values.
-    \param waits Pointer to array of wait values.
-    \param signal Pointer to signal value or NULL.
-*/
-extern "C" int OFFLOAD_OFFLOAD1(
-    OFFLOAD o,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void** waits,
-    const void** signal
-);
-
-/*! \fn OFFLOAD_OFFLOAD2
-    \brief Run function on target using interface for new data persistence.
-    \param o Offload descriptor created by OFFLOAD_TARGET_ACQUIRE.
-    \param name Name of offload entry point.
-    \param is_empty If no code to execute (e.g. offload_transfer)
-    \param num_vars Number of variable descriptors.
-    \param vars Pointer to VarDesc array.
-    \param vars2 Pointer to VarDesc2 array.
-    \param num_waits Number of "wait" values.
-    \param waits Pointer to array of wait values.
-    \param signal Pointer to signal value or NULL.
-    \param entry_id A signature for the function doing the offload.
-    \param stack_addr The stack frame address of the function doing offload.
-*/
-extern "C" int OFFLOAD_OFFLOAD2(
-    OFFLOAD o,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void** waits,
-    const void** signal,
-    int entry_id,
-    const void *stack_addr
-);
-
-// Run function on target (obsolete).
-// @param o    OFFLOAD object
-// @param name function name
-extern "C" int OFFLOAD_OFFLOAD(
-    OFFLOAD o,
-    const char *name,
-    int is_empty,
-    int num_vars,
-    VarDesc *vars,
-    VarDesc2 *vars2,
-    int num_waits,
-    const void** waits,
-    const void* signal,
-    int entry_id = 0,
-    const void *stack_addr = NULL
-);
-
-// Global counter on host.
-// This variable is used if P2OPT_offload_do_data_persistence == 2.
-// The variable used to identify offload constructs contained in one procedure.
-// Call to OFFLOAD_CALL_COUNT() is inserted at HOST on entry of the routine.
-extern "C" int  OFFLOAD_CALL_COUNT();
-
-#endif // COMPILER_IF_HOST_H_INCLUDED

offload/src/compiler_if_target.cpp

@@ -1,44 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "compiler_if_target.h"
-
-extern "C" void OFFLOAD_TARGET_ENTER(
-    OFFLOAD ofld,
-    int vars_total,
-    VarDesc *vars,
-    VarDesc2 *vars2
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p, %d, %p, %p)\n", __func__, ofld,
-                        vars_total, vars, vars2);
-    ofld->merge_var_descs(vars, vars2, vars_total);
-    ofld->scatter_copyin_data();
-}
-
-extern "C" void OFFLOAD_TARGET_LEAVE(
-    OFFLOAD ofld
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ofld);
-    ofld->gather_copyout_data();
-}
-
-extern "C" void OFFLOAD_TARGET_MAIN(void)
-{
-    // initialize target part
-    __offload_target_init();
-
-    // pass control to COI
-    PipelineStartExecutingRunFunctions();
-    ProcessWaitForShutdown();
-
-    OFFLOAD_DEBUG_TRACE(2, "Exiting main...\n");
-}

offload/src/compiler_if_target.h

@@ -1,50 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*! \file
-    \brief The interface between compiler-generated target code and runtime library
-*/
-
-#ifndef COMPILER_IF_TARGET_H_INCLUDED
-#define COMPILER_IF_TARGET_H_INCLUDED
-
-#include "offload_target.h"
-
-#define OFFLOAD_TARGET_ENTER            OFFLOAD_PREFIX(target_enter)
-#define OFFLOAD_TARGET_LEAVE            OFFLOAD_PREFIX(target_leave)
-#define OFFLOAD_TARGET_MAIN             OFFLOAD_PREFIX(target_main)
-
-/*! \fn OFFLOAD_TARGET_ENTER
-    \brief Fill in variable addresses using VarDesc array.
-    \brief Then call back the runtime library to fetch data.
-    \param ofld         Offload descriptor created by runtime.
-    \param var_desc_num Number of variable descriptors.
-    \param var_desc     Pointer to VarDesc array.
-    \param var_desc2    Pointer to VarDesc2 array.
-*/
-extern "C" void OFFLOAD_TARGET_ENTER(
-    OFFLOAD ofld,
-    int var_desc_num,
-    VarDesc *var_desc,
-    VarDesc2 *var_desc2
-);
-
-/*! \fn OFFLOAD_TARGET_LEAVE
-    \brief Call back the runtime library to gather outputs using VarDesc array.
-    \param ofld Offload descriptor created by OFFLOAD_TARGET_ACQUIRE.
-*/
-extern "C" void OFFLOAD_TARGET_LEAVE(
-    OFFLOAD ofld
-);
-
-// Entry point for the target application.
-extern "C" void OFFLOAD_TARGET_MAIN(void);
-
-#endif // COMPILER_IF_TARGET_H_INCLUDED

offload/src/dv_util.cpp

@@ -1,131 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_common.h"
-
-bool __dv_is_contiguous(const ArrDesc *dvp)
-{
-    if (dvp->Flags & ArrDescFlagsContiguous) {
-        return true;
-    }
-
-    if (dvp->Rank != 0) {
-        if (dvp->Dim[0].Mult != dvp->Len) {
-            return false;
-        }
-        for (int i = 1; i < dvp->Rank; i++) {
-            if (dvp->Dim[i].Mult !=
-                dvp->Dim[i-1].Extent * dvp->Dim[i-1].Mult) {
-                return false;
-            }
-        }
-    }
-    return true;
-}
-
-bool __dv_is_allocated(const ArrDesc *dvp)
-{
-    return (dvp->Flags & ArrDescFlagsDefined);
-}
-
-uint64_t __dv_data_length(const ArrDesc *dvp)
-{
-    uint64_t size;
-
-    if (dvp->Rank == 0) {
-        size = dvp->Len;
-        return size;
-    }
-
-    size = dvp->Len;
-    for (int i = 0; i < dvp->Rank; ++i) {
-        size += (dvp->Dim[i].Extent-1) * dvp->Dim[i].Mult;
-    }
-    return size;
-}
-
-uint64_t __dv_data_length(const ArrDesc *dvp, int64_t count)
-{
-    if (dvp->Rank == 0) {
-        return count;
-    }
-
-    return count * dvp->Dim[0].Mult;
-}
-
-// Create CeanReadRanges data for reading contiguous ranges of
-// noncontiguous array defined by the argument
-CeanReadRanges * init_read_ranges_dv(const ArrDesc *dvp)
-{
-    int64_t         len;
-    int             count;
-    int             rank = dvp->Rank;
-    CeanReadRanges *res = NULL;
-
-    if (rank != 0) {
-        int i = 0;
-        len = dvp->Len;
-        if (dvp->Dim[0].Mult == len) {
-            for (i = 1; i < rank; i++) {
-                len *= dvp->Dim[i-1].Extent;
-                if (dvp->Dim[i].Mult != len) {
-                    break;
-                }
-            }
-        }
-        res = (CeanReadRanges *)malloc(
-            sizeof(CeanReadRanges) + (rank - i) * sizeof(CeanReadDim));
-        res -> last_noncont_ind = rank - i - 1;
-        count = 1;
-        for (; i < rank; i++) {
-            res->Dim[rank - i - 1].count = count;
-            res->Dim[rank - i - 1].size = dvp->Dim[i].Mult;
-            count *= dvp->Dim[i].Extent;
-        }
-        res -> range_max_number = count;
-        res -> range_size = len;
-        res -> ptr = (void*)dvp->Base;
-        res -> current_number = 0;
-        res -> init_offset = 0;
-    }
-    return res;
-}
-
-#if OFFLOAD_DEBUG > 0
-void __dv_desc_dump(const char *name, const ArrDesc *dvp)
-{
-    OFFLOAD_TRACE(3, "%s DV %p\n", name, dvp);
-
-    if (dvp != 0) {
-        OFFLOAD_TRACE(3,
-                      "    dv->Base   = 0x%lx\n"
-                      "    dv->Len    = 0x%lx\n"
-                      "    dv->Offset = 0x%lx\n"
-                      "    dv->Flags  = 0x%lx\n"
-                      "    dv->Rank   = 0x%lx\n"
-                      "    dv->Resrvd = 0x%lx\n",
-                      dvp->Base,
-                      dvp->Len,
-                      dvp->Offset,
-                      dvp->Flags,
-                      dvp->Rank,
-                      dvp->Reserved);
-
-        for (int i = 0 ; i < dvp->Rank; i++) {
-            OFFLOAD_TRACE(3,
-                          "    (%d) Extent=%ld, Multiplier=%ld, LowerBound=%ld\n",
-                          i,
-                          dvp->Dim[i].Extent,
-                          dvp->Dim[i].Mult,
-                          dvp->Dim[i].LowerBound);
-        }
-    }
-}
-#endif // OFFLOAD_DEBUG > 0

offload/src/dv_util.h

@@ -1,63 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef DV_UTIL_H_INCLUDED
-#define DV_UTIL_H_INCLUDED
-
-#include <stdint.h>
-
-// Dope vector declarations
-#define ArrDescMaxArrayRank         31
-
-// Dope vector flags
-#define ArrDescFlagsDefined         1
-#define ArrDescFlagsNodealloc       2
-#define ArrDescFlagsContiguous      4
-
-typedef int64_t dv_size;
-
-typedef struct DimDesc {
-    dv_size        Extent;      // Number of elements in this dimension
-    dv_size        Mult;        // Multiplier for this dimension.
-                                // The number of bytes between successive
-                                // elements in this dimension.
-    dv_size        LowerBound;  // LowerBound of this dimension
-} DimDesc ;
-
-typedef struct ArrDesc {
-    dv_size        Base;        // Base address
-    dv_size        Len;         // Length of data type, used only for
-                                // character strings.
-    dv_size        Offset;
-    dv_size        Flags;       // Flags
-    dv_size        Rank;        // Rank of pointer
-    dv_size        Reserved;    // reserved for openmp requests
-    DimDesc Dim[ArrDescMaxArrayRank];
-} ArrDesc ;
-
-typedef ArrDesc* pArrDesc;
-
-bool __dv_is_contiguous(const ArrDesc *dvp);
-
-bool __dv_is_allocated(const ArrDesc *dvp);
-
-uint64_t __dv_data_length(const ArrDesc *dvp);
-
-uint64_t __dv_data_length(const ArrDesc *dvp, int64_t nelems);
-
-CeanReadRanges * init_read_ranges_dv(const ArrDesc *dvp);
-
-#if OFFLOAD_DEBUG > 0
-void    __dv_desc_dump(const char *name, const ArrDesc *dvp);
-#else // OFFLOAD_DEBUG
-#define __dv_desc_dump(name, dvp)
-#endif // OFFLOAD_DEBUG
-
-#endif // DV_UTIL_H_INCLUDED

offload/src/liboffload_error.c

@@ -1,452 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include <stdio.h>
-#include <stdarg.h>
-#ifndef va_copy
-#define va_copy(dst, src) ((dst) = (src))
-#endif
-
-#include "liboffload_msg.h"
-
-#include "liboffload_error_codes.h"
-
-/***********************************************/
-/* error-handling function, liboffload_error_support */
-/***********************************************/
-
-void __liboffload_error_support(error_types input_tag, ...)
-{
-    va_list args;
-    va_start(args, input_tag);
-
-    switch (input_tag) {
-        case c_device_is_not_available:
-            write_message(stderr, msg_c_device_is_not_available, args);
-            break;
-        case c_invalid_device_number:
-            write_message(stderr, msg_c_invalid_device_number, args);
-            break;
-        case c_send_func_ptr:
-            write_message(stderr, msg_c_send_func_ptr, args);
-            break;
-        case c_receive_func_ptr:
-            write_message(stderr, msg_c_receive_func_ptr, args);
-            break;
-        case c_offload_malloc:
-            write_message(stderr, msg_c_offload_malloc, args);
-            break;
-        case c_offload1:
-            write_message(stderr, msg_c_offload1, args);
-            break;
-        case c_unknown_var_type:
-            write_message(stderr, c_unknown_var_type, args);
-            break;
-        case c_invalid_env_var_value:
-            write_message(stderr, msg_c_invalid_env_var_value, args);
-            break;
-        case c_invalid_env_var_int_value:
-            write_message(stderr, msg_c_invalid_env_var_int_value, args);
-            break;
-        case c_invalid_env_report_value:
-            write_message(stderr, msg_c_invalid_env_report_value, args);
-            break;
-        case c_offload_signaled1:
-            write_message(stderr, msg_c_offload_signaled1, args);
-            break;
-        case c_offload_signaled2:
-            write_message(stderr, msg_c_offload_signaled2, args);
-            break;
-        case c_myowrapper_checkresult:
-            write_message(stderr, msg_c_myowrapper_checkresult, args);
-            break;
-        case c_myotarget_checkresult:
-            write_message(stderr, msg_c_myotarget_checkresult, args);
-            break;
-        case c_offload_descriptor_offload:
-            write_message(stderr, msg_c_offload_descriptor_offload, args);
-            break;
-        case c_merge_var_descs1:
-            write_message(stderr, msg_c_merge_var_descs1, args);
-            break;
-        case c_merge_var_descs2:
-            write_message(stderr, msg_c_merge_var_descs2, args);
-            break;
-        case c_mic_parse_env_var_list1:
-            write_message(stderr, msg_c_mic_parse_env_var_list1, args);
-            break;
-        case c_mic_parse_env_var_list2:
-            write_message(stderr, msg_c_mic_parse_env_var_list2, args);
-            break;
-        case c_mic_process_exit_ret:
-            write_message(stderr, msg_c_mic_process_exit_ret, args);
-            break;
-        case c_mic_process_exit_sig:
-            write_message(stderr, msg_c_mic_process_exit_sig, args);
-            break;
-        case c_mic_process_exit:
-            write_message(stderr, msg_c_mic_process_exit, args);
-            break;
-        case c_mic_init3:
-            write_message(stderr, msg_c_mic_init3, args);
-            break;
-        case c_mic_init4:
-            write_message(stderr, msg_c_mic_init4, args);
-            break;
-        case c_mic_init5:
-            write_message(stderr, msg_c_mic_init5, args);
-            break;
-        case c_mic_init6:
-            write_message(stderr, msg_c_mic_init6, args);
-            break;
-        case c_no_static_var_data:
-            write_message(stderr, msg_c_no_static_var_data, args);
-            break;
-        case c_no_ptr_data:
-            write_message(stderr, msg_c_no_ptr_data, args);
-            break;
-        case c_get_engine_handle:
-            write_message(stderr, msg_c_get_engine_handle, args);
-            break;
-        case c_get_engine_index:
-            write_message(stderr, msg_c_get_engine_index, args);
-            break;
-        case c_process_create:
-            write_message(stderr, msg_c_process_create, args);
-            break;
-        case c_process_wait_shutdown:
-            write_message(stderr, msg_c_process_wait_shutdown, args);
-            break;
-        case c_process_proxy_flush:
-            write_message(stderr, msg_c_process_proxy_flush, args);
-            break;
-        case c_process_get_func_handles:
-            write_message(stderr, msg_c_process_get_func_handles, args);
-            break;
-        case c_load_library:
-            write_message(stderr, msg_c_load_library, args);
-            break;
-        case c_coipipe_max_number:
-            write_message(stderr, msg_c_coi_pipeline_max_number, args);
-            break;
-        case c_pipeline_create:
-            write_message(stderr, msg_c_pipeline_create, args);
-            break;
-        case c_pipeline_run_func:
-            write_message(stderr, msg_c_pipeline_run_func, args);
-            break;
-        case c_pipeline_start_run_funcs:
-            write_message(stderr, msg_c_pipeline_start_run_funcs, args);
-            break;
-        case c_buf_create:
-            write_message(stderr, msg_c_buf_create, args);
-            break;
-        case c_buf_create_out_of_mem:
-            write_message(stderr, msg_c_buf_create_out_of_mem, args);
-            break;
-        case c_buf_create_from_mem:
-            write_message(stderr, msg_c_buf_create_from_mem, args);
-            break;
-        case c_buf_destroy:
-            write_message(stderr, msg_c_buf_destroy, args);
-            break;
-        case c_buf_map:
-            write_message(stderr, msg_c_buf_map, args);
-            break;
-        case c_buf_unmap:
-            write_message(stderr, msg_c_buf_unmap, args);
-            break;
-        case c_buf_read:
-            write_message(stderr, msg_c_buf_read, args);
-            break;
-        case c_buf_write:
-            write_message(stderr, msg_c_buf_write, args);
-            break;
-        case c_buf_copy:
-            write_message(stderr, msg_c_buf_copy, args);
-            break;
-        case c_buf_get_address:
-            write_message(stderr, msg_c_buf_get_address, args);
-            break;
-        case c_buf_add_ref:
-            write_message(stderr, msg_c_buf_add_ref, args);
-            break;
-        case c_buf_release_ref:
-            write_message(stderr, msg_c_buf_release_ref, args);
-            break;
-        case c_buf_set_state:
-            write_message(stderr, msg_c_buf_set_state, args);
-            break;
-        case c_event_wait:
-            write_message(stderr, msg_c_event_wait, args);
-            break;
-        case c_zero_or_neg_ptr_len:
-            write_message(stderr, msg_c_zero_or_neg_ptr_len, args);
-            break;
-        case c_zero_or_neg_transfer_size:
-            write_message(stderr, msg_c_zero_or_neg_transfer_size, args);
-            break;
-        case c_bad_ptr_mem_range:
-            write_message(stderr, msg_c_bad_ptr_mem_range, args);
-            break;
-        case c_different_src_and_dstn_sizes:
-            write_message(stderr, msg_c_different_src_and_dstn_sizes, args);
-            break;
-        case c_ranges_dont_match:
-            write_message(stderr, msg_c_ranges_dont_match, args);
-            break;
-        case c_destination_is_over:
-            write_message(stderr, msg_c_destination_is_over, args);
-            break;
-        case c_slice_of_noncont_array:
-            write_message(stderr, msg_c_slice_of_noncont_array, args);
-            break;
-        case c_non_contiguous_dope_vector:
-            write_message(stderr, msg_c_non_contiguous_dope_vector, args);
-            break;
-        case c_pointer_array_mismatch:
-            write_message(stderr, msg_c_pointer_array_mismatch, args);
-            break;
-        case c_omp_invalid_device_num_env:
-            write_message(stderr, msg_c_omp_invalid_device_num_env, args);
-            break;
-        case c_omp_invalid_device_num:
-            write_message(stderr, msg_c_omp_invalid_device_num, args);
-            break;
-        case c_unknown_binary_type:
-            write_message(stderr, msg_c_unknown_binary_type, args);
-            break;
-        case c_multiple_target_exes:
-            write_message(stderr, msg_c_multiple_target_exes, args);
-            break;
-        case c_no_target_exe:
-            write_message(stderr, msg_c_no_target_exe, args);
-            break;
-        case c_report_unknown_timer_node:
-            write_message(stderr, msg_c_report_unknown_timer_node, args);
-            break;
-        case c_report_unknown_trace_node:
-            write_message(stderr, msg_c_report_unknown_trace_node, args);
-            break;
-    }
-    va_end(args);
-}
-
-char const * report_get_message_str(error_types input_tag)
-{
-    switch (input_tag) {
-        case c_report_title:
-            return (offload_get_message_str(msg_c_report_title));
-        case c_report_from_file:
-            return (offload_get_message_str(msg_c_report_from_file));
-        case c_report_offload:
-            return (offload_get_message_str(msg_c_report_offload));
-        case c_report_mic:
-            return (offload_get_message_str(msg_c_report_mic));
-        case c_report_file:
-            return (offload_get_message_str(msg_c_report_file));
-        case c_report_line:
-            return (offload_get_message_str(msg_c_report_line));
-        case c_report_host:
-            return (offload_get_message_str(msg_c_report_host));
-        case c_report_tag:
-            return (offload_get_message_str(msg_c_report_tag));
-        case c_report_cpu_time:
-            return (offload_get_message_str(msg_c_report_cpu_time));
-        case c_report_seconds:
-            return (offload_get_message_str(msg_c_report_seconds));
-        case c_report_cpu_to_mic_data:
-            return (offload_get_message_str(msg_c_report_cpu_to_mic_data));
-        case c_report_bytes:
-            return (offload_get_message_str(msg_c_report_bytes));
-        case c_report_mic_time:
-            return (offload_get_message_str(msg_c_report_mic_time));
-        case c_report_mic_to_cpu_data:
-            return (offload_get_message_str(msg_c_report_mic_to_cpu_data));
-        case c_report_compute:
-            return (offload_get_message_str(msg_c_report_compute));
-        case c_report_copyin_data:
-            return (offload_get_message_str(msg_c_report_copyin_data));
-        case c_report_copyout_data:
-            return (offload_get_message_str(msg_c_report_copyout_data));
-        case c_report_create_buf_host:
-            return (offload_get_message_str(c_report_create_buf_host));
-        case c_report_create_buf_mic:
-            return (offload_get_message_str(msg_c_report_create_buf_mic));
-        case c_report_destroy:
-            return (offload_get_message_str(msg_c_report_destroy));
-        case c_report_gather_copyin_data:
-            return (offload_get_message_str(msg_c_report_gather_copyin_data));
-        case c_report_gather_copyout_data:
-            return (offload_get_message_str(msg_c_report_gather_copyout_data));
-        case c_report_state_signal:
-            return (offload_get_message_str(msg_c_report_state_signal));
-        case c_report_signal:
-            return (offload_get_message_str(msg_c_report_signal));
-        case c_report_wait:
-            return (offload_get_message_str(msg_c_report_wait));
-        case c_report_init:
-            return (offload_get_message_str(msg_c_report_init));
-        case c_report_init_func:
-            return (offload_get_message_str(msg_c_report_init_func));
-        case c_report_logical_card:
-            return (offload_get_message_str(msg_c_report_logical_card));
-        case c_report_mic_myo_fptr:
-            return (offload_get_message_str(msg_c_report_mic_myo_fptr));
-        case c_report_mic_myo_shared:
-            return (offload_get_message_str(msg_c_report_mic_myo_shared));
-        case c_report_myoacquire:
-            return (offload_get_message_str(msg_c_report_myoacquire));
-        case c_report_myofini:
-            return (offload_get_message_str(msg_c_report_myofini));
-        case c_report_myoinit:
-            return (offload_get_message_str(msg_c_report_myoinit));
-        case c_report_myoregister:
-            return (offload_get_message_str(msg_c_report_myoregister));
-        case c_report_myorelease:
-            return (offload_get_message_str(msg_c_report_myorelease));
-        case c_report_myosharedalignedfree:
-            return (
-                offload_get_message_str(msg_c_report_myosharedalignedfree));
-        case c_report_myosharedalignedmalloc:
-            return (
-                offload_get_message_str(msg_c_report_myosharedalignedmalloc));
-        case c_report_myosharedfree:
-            return (offload_get_message_str(msg_c_report_myosharedfree));
-        case c_report_myosharedmalloc:
-            return (offload_get_message_str(msg_c_report_myosharedmalloc));
-        case c_report_physical_card:
-            return (offload_get_message_str(msg_c_report_physical_card));
-        case c_report_receive_pointer_data:
-            return (
-                offload_get_message_str(msg_c_report_receive_pointer_data));
-        case c_report_received_pointer_data:
-            return (
-                offload_get_message_str(msg_c_report_received_pointer_data));
-        case c_report_register:
-            return (offload_get_message_str(msg_c_report_register));
-        case c_report_scatter_copyin_data:
-            return (offload_get_message_str(msg_c_report_scatter_copyin_data));
-        case c_report_scatter_copyout_data:
-            return (
-                offload_get_message_str(msg_c_report_scatter_copyout_data));
-        case c_report_send_pointer_data:
-            return (offload_get_message_str(msg_c_report_send_pointer_data));
-        case c_report_sent_pointer_data:
-            return (offload_get_message_str(msg_c_report_sent_pointer_data));
-        case c_report_start:
-            return (offload_get_message_str(msg_c_report_start));
-        case c_report_start_target_func:
-            return (offload_get_message_str(msg_c_report_start_target_func));
-        case c_report_state:
-            return (offload_get_message_str(msg_c_report_state));
-        case c_report_unregister:
-            return (offload_get_message_str(msg_c_report_unregister));
-        case c_report_var:
-            return (offload_get_message_str(msg_c_report_var));
-
-        default:
-            LIBOFFLOAD_ERROR(c_report_unknown_trace_node);
-            abort();
-    }
-}
-
-char const * report_get_host_stage_str(int i)
-{
-    switch (i) {
-        case c_offload_host_total_offload:
-            return (
-               offload_get_message_str(msg_c_report_host_total_offload_time));
-        case c_offload_host_initialize:
-            return (offload_get_message_str(msg_c_report_host_initialize));
-        case c_offload_host_target_acquire:
-            return (
-                offload_get_message_str(msg_c_report_host_target_acquire));
-        case c_offload_host_wait_deps:
-            return (offload_get_message_str(msg_c_report_host_wait_deps));
-        case c_offload_host_setup_buffers:
-            return (offload_get_message_str(msg_c_report_host_setup_buffers));
-        case c_offload_host_alloc_buffers:
-            return (offload_get_message_str(msg_c_report_host_alloc_buffers));
-        case c_offload_host_setup_misc_data:
-            return (
-                offload_get_message_str(msg_c_report_host_setup_misc_data));
-        case c_offload_host_alloc_data_buffer:
-            return (
-                offload_get_message_str(msg_c_report_host_alloc_data_buffer));
-        case c_offload_host_send_pointers:
-            return (offload_get_message_str(msg_c_report_host_send_pointers));
-        case c_offload_host_gather_inputs:
-            return (offload_get_message_str(msg_c_report_host_gather_inputs));
-        case c_offload_host_map_in_data_buffer:
-            return (
-                offload_get_message_str(msg_c_report_host_map_in_data_buffer));
-        case c_offload_host_unmap_in_data_buffer:
-            return (offload_get_message_str(
-                msg_c_report_host_unmap_in_data_buffer));
-        case c_offload_host_start_compute:
-            return (offload_get_message_str(msg_c_report_host_start_compute));
-        case c_offload_host_wait_compute:
-            return (offload_get_message_str(msg_c_report_host_wait_compute));
-        case c_offload_host_start_buffers_reads:
-            return (offload_get_message_str(
-                msg_c_report_host_start_buffers_reads));
-        case c_offload_host_scatter_outputs:
-            return (
-                offload_get_message_str(msg_c_report_host_scatter_outputs));
-        case c_offload_host_map_out_data_buffer:
-            return (offload_get_message_str(
-                msg_c_report_host_map_out_data_buffer));
-        case c_offload_host_unmap_out_data_buffer:
-            return (offload_get_message_str(
-                msg_c_report_host_unmap_out_data_buffer));
-        case c_offload_host_wait_buffers_reads:
-            return (
-                offload_get_message_str(msg_c_report_host_wait_buffers_reads));
-        case c_offload_host_destroy_buffers:
-            return (
-                offload_get_message_str(msg_c_report_host_destroy_buffers));
-        default:
-            LIBOFFLOAD_ERROR(c_report_unknown_timer_node);
-            abort();
-    }
-}
-
-char const * report_get_target_stage_str(int i)
-{
-    switch (i) {
-        case c_offload_target_total_time:
-            return (offload_get_message_str(msg_c_report_target_total_time));
-        case c_offload_target_descriptor_setup:
-            return (
-                offload_get_message_str(msg_c_report_target_descriptor_setup));
-        case c_offload_target_func_lookup:
-            return (offload_get_message_str(msg_c_report_target_func_lookup));
-        case c_offload_target_func_time:
-            return (offload_get_message_str(msg_c_report_target_func_time));
-        case c_offload_target_scatter_inputs:
-            return (
-                offload_get_message_str(msg_c_report_target_scatter_inputs));
-        case c_offload_target_add_buffer_refs:
-            return (
-                offload_get_message_str(msg_c_report_target_add_buffer_refs));
-        case c_offload_target_compute:
-            return (offload_get_message_str(msg_c_report_target_compute));
-        case c_offload_target_gather_outputs:
-            return (offload_get_message_str
-                (msg_c_report_target_gather_outputs));
-        case c_offload_target_release_buffer_refs:
-            return (offload_get_message_str(
-                msg_c_report_target_release_buffer_refs));
-        default:
-            LIBOFFLOAD_ERROR(c_report_unknown_timer_node);
-            abort();
-    }
-}

offload/src/liboffload_error_codes.h

@@ -1,276 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#if !defined(LIBOFFLOAD_ERROR_CODES_H)
-#define LIBOFFLOAD_ERROR_CODES_H
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-typedef enum
-{
-    c_device_is_not_available = 0,
-    c_invalid_device_number,
-    c_offload1,
-    c_unknown_var_type,
-    c_send_func_ptr,
-    c_receive_func_ptr,
-    c_offload_malloc,
-    c_invalid_env_var_value,
-    c_invalid_env_var_int_value,
-    c_invalid_env_report_value,
-    c_offload_signaled1,
-    c_offload_signaled2,
-    c_myotarget_checkresult,
-    c_myowrapper_checkresult,
-    c_offload_descriptor_offload,
-    c_merge_var_descs1,
-    c_merge_var_descs2,
-    c_mic_parse_env_var_list1,
-    c_mic_parse_env_var_list2,
-    c_mic_process_exit_ret,
-    c_mic_process_exit_sig,
-    c_mic_process_exit,
-    c_mic_init3,
-    c_mic_init4,
-    c_mic_init5,
-    c_mic_init6,
-    c_no_static_var_data,
-    c_no_ptr_data,
-    c_get_engine_handle,
-    c_get_engine_index,
-    c_process_create,
-    c_process_get_func_handles,
-    c_process_wait_shutdown,
-    c_process_proxy_flush,
-    c_load_library,
-    c_pipeline_create,
-    c_pipeline_run_func,
-    c_pipeline_start_run_funcs,
-    c_buf_create,
-    c_buf_create_out_of_mem,
-    c_buf_create_from_mem,
-    c_buf_destroy,
-    c_buf_map,
-    c_buf_unmap,
-    c_buf_read,
-    c_buf_write,
-    c_buf_copy,
-    c_buf_get_address,
-    c_buf_add_ref,
-    c_buf_release_ref,
-    c_buf_set_state,
-    c_event_wait,
-    c_zero_or_neg_ptr_len,
-    c_zero_or_neg_transfer_size,
-    c_bad_ptr_mem_range,
-    c_different_src_and_dstn_sizes,
-    c_ranges_dont_match,
-    c_destination_is_over,
-    c_slice_of_noncont_array,
-    c_non_contiguous_dope_vector,
-    c_pointer_array_mismatch,
-    c_omp_invalid_device_num_env,
-    c_omp_invalid_device_num,
-    c_unknown_binary_type,
-    c_multiple_target_exes,
-    c_no_target_exe,
-    c_report_host,
-    c_report_target,
-    c_report_title,
-    c_report_from_file,
-    c_report_file,
-    c_report_line,
-    c_report_tag,
-    c_report_seconds,
-    c_report_bytes,
-    c_report_mic,
-    c_report_cpu_time,
-    c_report_cpu_to_mic_data,
-    c_report_mic_time,
-    c_report_mic_to_cpu_data,
-    c_report_unknown_timer_node,
-    c_report_unknown_trace_node,
-    c_report_offload,
-    c_report_w_tag,
-    c_report_state,
-    c_report_start,
-    c_report_init,
-    c_report_logical_card,
-    c_report_physical_card,
-    c_report_register,
-    c_report_init_func,
-    c_report_create_buf_host,
-    c_report_create_buf_mic,
-    c_report_send_pointer_data,
-    c_report_sent_pointer_data,
-    c_report_gather_copyin_data,
-    c_report_copyin_data,
-    c_report_state_signal,
-    c_report_signal,
-    c_report_wait,
-    c_report_compute,
-    c_report_receive_pointer_data,
-    c_report_received_pointer_data,
-    c_report_start_target_func,
-    c_report_var,
-    c_report_scatter_copyin_data,
-    c_report_gather_copyout_data,
-    c_report_scatter_copyout_data,
-    c_report_copyout_data,
-    c_report_unregister,
-    c_report_destroy,
-    c_report_myoinit,
-    c_report_myoregister,
-    c_report_myofini,
-    c_report_mic_myo_shared,
-    c_report_mic_myo_fptr,
-    c_report_myosharedmalloc,
-    c_report_myosharedfree,
-    c_report_myosharedalignedmalloc,
-    c_report_myosharedalignedfree,
-    c_report_myoacquire,
-    c_report_myorelease,
-    c_coipipe_max_number
-} error_types;
-
-enum OffloadHostPhase {
-    // Total time on host for entire offload
-    c_offload_host_total_offload = 0,
-
-    // Time to load target binary
-    c_offload_host_initialize,
-
-    // Time to acquire lrb availability dynamically
-    c_offload_host_target_acquire,
-
-    // Time to wait for dependencies
-    c_offload_host_wait_deps,
-
-    // Time to allocate pointer buffers, initiate writes for pointers
-    // and calculate size of copyin/copyout buffer
-    c_offload_host_setup_buffers,
-
-    // Time to allocate pointer buffers
-    c_offload_host_alloc_buffers,
-
-    // Time to initialize misc data
-    c_offload_host_setup_misc_data,
-
-    // Time to allocate copyin/copyout buffer
-    c_offload_host_alloc_data_buffer,
-
-    // Time to initiate writes from host pointers to buffers
-    c_offload_host_send_pointers,
-
-    // Time to Gather IN data of offload into buffer
-    c_offload_host_gather_inputs,
-
-    // Time to map buffer
-    c_offload_host_map_in_data_buffer,
-
-    // Time to unmap buffer
-    c_offload_host_unmap_in_data_buffer,
-
-    // Time to start remote function call that does computation on lrb
-    c_offload_host_start_compute,
-
-    // Time to wait for compute to finish
-    c_offload_host_wait_compute,
-
-    // Time to initiate reads from pointer buffers
-    c_offload_host_start_buffers_reads,
-
-    // Time to update host variabels with OUT data from buffer
-    c_offload_host_scatter_outputs,
-
-    // Time to map buffer
-    c_offload_host_map_out_data_buffer,
-
-    // Time to unmap buffer
-    c_offload_host_unmap_out_data_buffer,
-
-    // Time to wait reads from buffers to finish
-    c_offload_host_wait_buffers_reads,
-
-    // Time to destroy buffers that are no longer needed
-    c_offload_host_destroy_buffers,
-
-    // LAST TIME MONITOR
-    c_offload_host_max_phase
-};
-
-enum OffloadTargetPhase {
-    // Total time spent on the target
-    c_offload_target_total_time = 0,
-
-    // Time to initialize offload descriptor
-    c_offload_target_descriptor_setup,
-
-    // Time to find target entry point in lookup table
-    c_offload_target_func_lookup,
-
-    // Total time spend executing offload entry
-    c_offload_target_func_time,
-
-    // Time to initialize target variables with IN values from buffer
-    c_offload_target_scatter_inputs,
-
-    // Time to add buffer reference for pointer buffers
-    c_offload_target_add_buffer_refs,
-
-    // Total time on lrb for computation
-    c_offload_target_compute,
-
-    // On lrb, time to copy OUT into buffer
-    c_offload_target_gather_outputs,
-
-    // Time to release buffer references
-    c_offload_target_release_buffer_refs,
-
-    // LAST TIME MONITOR
-    c_offload_target_max_phase
-};
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-void __liboffload_error_support(error_types input_tag, ...);
-void __liboffload_report_support(error_types input_tag, ...);
-char const *offload_get_message_str(int msgCode);
-char const * report_get_message_str(error_types input_tag);
-char const * report_get_host_stage_str(int i);
-char const * report_get_target_stage_str(int i);
-#ifdef __cplusplus
-}
-#endif
-
-#define test_msg_cat(nm, msg) \
-    fprintf(stderr, "\t TEST for %s \n \t", nm); \
-    __liboffload_error_support(msg);
-
-#define test_msg_cat1(nm, msg, ...) \
-    fprintf(stderr, "\t TEST for %s \n \t", nm); \
-    __liboffload_error_support(msg, __VA_ARGS__);
-
-void write_message(FILE * file, int msgCode, va_list args_p);
-
-#define LIBOFFLOAD_ERROR __liboffload_error_support
-
-#ifdef TARGET_WINNT
-#define LIBOFFLOAD_ABORT \
-         _set_abort_behavior(0, _WRITE_ABORT_MSG); \
-         abort()
-#else
-#define LIBOFFLOAD_ABORT \
-         abort()
-#endif
-
-#endif // !defined(LIBOFFLOAD_ERROR_CODES_H)

offload/src/liboffload_msg.c

@@ -1,35 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-
-#include <stdio.h>
-
-// ===========================================================================
-// Bring in the static string table and the enumerations for indexing into
-// it.
-// ===========================================================================
-
-#include "liboffload_msg.h"
-
-# define DYNART_STDERR_PUTS(__message_text__) fputs((__message_text__),stderr)
-
-// ===========================================================================
-// Now the code for accessing the message catalogs
-// ===========================================================================
-
-
-    void write_message(FILE * file, int msgCode) {
-        fputs(MESSAGE_TABLE_NAME[ msgCode ], file);
-        fflush(file);
-    }
-
-    char const *offload_get_message_str(int msgCode) {
-        return MESSAGE_TABLE_NAME[ msgCode ];
-    }

offload/src/liboffload_msg.h

@@ -1,326 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-// file: liboffload_msg.h
-enum {
-	__dummy__ = 0,
-	msg_c_device_is_not_available,
-	msg_c_invalid_device_number,
-	msg_c_send_func_ptr,
-	msg_c_receive_func_ptr,
-	msg_c_offload_malloc,
-	msg_c_offload1,
-	msg_c_unknown_var_type,
-	msg_c_invalid_env_var_value,
-	msg_c_invalid_env_var_int_value,
-	msg_c_invalid_env_report_value,
-	msg_c_offload_signaled1,
-	msg_c_offload_signaled2,
-	msg_c_myowrapper_checkresult,
-	msg_c_myotarget_checkresult,
-	msg_c_offload_descriptor_offload,
-	msg_c_merge_var_descs1,
-	msg_c_merge_var_descs2,
-	msg_c_mic_parse_env_var_list1,
-	msg_c_mic_parse_env_var_list2,
-	msg_c_mic_process_exit_ret,
-	msg_c_mic_process_exit_sig,
-	msg_c_mic_process_exit,
-	msg_c_mic_init3,
-	msg_c_mic_init4,
-	msg_c_mic_init5,
-	msg_c_mic_init6,
-	msg_c_no_static_var_data,
-	msg_c_no_ptr_data,
-	msg_c_get_engine_handle,
-	msg_c_get_engine_index,
-	msg_c_process_create,
-	msg_c_process_get_func_handles,
-	msg_c_process_wait_shutdown,
-	msg_c_process_proxy_flush,
-	msg_c_load_library,
-	msg_c_pipeline_create,
-	msg_c_pipeline_run_func,
-	msg_c_pipeline_start_run_funcs,
-	msg_c_buf_create,
-	msg_c_buf_create_out_of_mem,
-	msg_c_buf_create_from_mem,
-	msg_c_buf_destroy,
-	msg_c_buf_map,
-	msg_c_buf_unmap,
-	msg_c_buf_read,
-	msg_c_buf_write,
-	msg_c_buf_copy,
-	msg_c_buf_get_address,
-	msg_c_buf_add_ref,
-	msg_c_buf_release_ref,
-	msg_c_buf_set_state,
-	msg_c_event_wait,
-	msg_c_zero_or_neg_ptr_len,
-	msg_c_zero_or_neg_transfer_size,
-	msg_c_bad_ptr_mem_range,
-	msg_c_different_src_and_dstn_sizes,
-	msg_c_non_contiguous_dope_vector,
-	msg_c_omp_invalid_device_num_env,
-	msg_c_omp_invalid_device_num,
-	msg_c_unknown_binary_type,
-	msg_c_multiple_target_exes,
-	msg_c_no_target_exe,
-	msg_c_report_unknown_timer_node,
-	msg_c_report_unknown_trace_node,
-	msg_c_report_host,
-	msg_c_report_mic,
-	msg_c_report_title,
-	msg_c_report_seconds,
-	msg_c_report_bytes,
-	msg_c_report_cpu_time,
-	msg_c_report_mic_time,
-	msg_c_report_tag,
-	msg_c_report_from_file,
-	msg_c_report_file,
-	msg_c_report_line,
-	msg_c_report_cpu_to_mic_data,
-	msg_c_report_mic_to_cpu_data,
-	msg_c_report_offload,
-	msg_c_report_w_tag,
-	msg_c_report_state,
-	msg_c_report_start,
-	msg_c_report_init,
-	msg_c_report_logical_card,
-	msg_c_report_physical_card,
-	msg_c_report_register,
-	msg_c_report_init_func,
-	msg_c_report_create_buf_host,
-	msg_c_report_create_buf_mic,
-	msg_c_report_send_pointer_data,
-	msg_c_report_sent_pointer_data,
-	msg_c_report_gather_copyin_data,
-	msg_c_report_copyin_data,
-	msg_c_report_state_signal,
-	msg_c_report_signal,
-	msg_c_report_wait,
-	msg_c_report_compute,
-	msg_c_report_receive_pointer_data,
-	msg_c_report_received_pointer_data,
-	msg_c_report_start_target_func,
-	msg_c_report_var,
-	msg_c_report_scatter_copyin_data,
-	msg_c_report_gather_copyout_data,
-	msg_c_report_scatter_copyout_data,
-	msg_c_report_copyout_data,
-	msg_c_report_unregister,
-	msg_c_report_destroy,
-	msg_c_report_myoinit,
-	msg_c_report_myoregister,
-	msg_c_report_myofini,
-	msg_c_report_mic_myo_shared,
-	msg_c_report_mic_myo_fptr,
-	msg_c_report_myosharedmalloc,
-	msg_c_report_myosharedfree,
-	msg_c_report_myosharedalignedmalloc,
-	msg_c_report_myosharedalignedfree,
-	msg_c_report_myoacquire,
-	msg_c_report_myorelease,
-	msg_c_report_host_total_offload_time,
-	msg_c_report_host_initialize,
-	msg_c_report_host_target_acquire,
-	msg_c_report_host_wait_deps,
-	msg_c_report_host_setup_buffers,
-	msg_c_report_host_alloc_buffers,
-	msg_c_report_host_setup_misc_data,
-	msg_c_report_host_alloc_data_buffer,
-	msg_c_report_host_send_pointers,
-	msg_c_report_host_gather_inputs,
-	msg_c_report_host_map_in_data_buffer,
-	msg_c_report_host_unmap_in_data_buffer,
-	msg_c_report_host_start_compute,
-	msg_c_report_host_wait_compute,
-	msg_c_report_host_start_buffers_reads,
-	msg_c_report_host_scatter_outputs,
-	msg_c_report_host_map_out_data_buffer,
-	msg_c_report_host_unmap_out_data_buffer,
-	msg_c_report_host_wait_buffers_reads,
-	msg_c_report_host_destroy_buffers,
-	msg_c_report_target_total_time,
-	msg_c_report_target_descriptor_setup,
-	msg_c_report_target_func_lookup,
-	msg_c_report_target_func_time,
-	msg_c_report_target_scatter_inputs,
-	msg_c_report_target_add_buffer_refs,
-	msg_c_report_target_compute,
-	msg_c_report_target_gather_outputs,
-	msg_c_report_target_release_buffer_refs,
-	msg_c_coi_pipeline_max_number,
-	msg_c_ranges_dont_match,
-	msg_c_destination_is_over,
-	msg_c_slice_of_noncont_array,
-	msg_c_pointer_array_mismatch,
-	lastMsg = 152,
-	firstMsg = 1
-};
-
-
-#if !defined(MESSAGE_TABLE_NAME)
-#    define MESSAGE_TABLE_NAME __liboffload_message_table
-#endif
-
-static char const * MESSAGE_TABLE_NAME[] = {
-	/*   0 __dummy__                               */ "Un-used message",
-	/*   1 msg_c_device_is_not_available           */ "offload error: cannot offload to MIC - device is not available",
-	/*   2 msg_c_invalid_device_number             */ "offload error: expected a number greater than or equal to -1",
-	/*   3 msg_c_send_func_ptr                     */ "offload error: cannot find function name for address %p",
-	/*   4 msg_c_receive_func_ptr                  */ "offload error: cannot find address of function %s",
-	/*   5 msg_c_offload_malloc                    */ "offload error: memory allocation failed (requested=%lld bytes, align %lld)",
-	/*   6 msg_c_offload1                          */ "offload error: device %d does not have a pending signal for wait(%p)",
-	/*   7 msg_c_unknown_var_type                  */ "offload error: unknown variable type %d",
-	/*   8 msg_c_invalid_env_var_value             */ "offload warning: ignoring invalid value specified for %s",
-	/*   9 msg_c_invalid_env_var_int_value         */ "offload warning: specify an integer value for %s",
-	/*  10 msg_c_invalid_env_report_value          */ "offload warning: ignoring %s setting; use a value in range 1-3",
-	/*  11 msg_c_offload_signaled1                 */ "offload error: invalid device number %d specified in _Offload_signaled",
-	/*  12 msg_c_offload_signaled2                 */ "offload error: invalid signal %p specified for _Offload_signaled",
-	/*  13 msg_c_myowrapper_checkresult            */ "offload error: %s failed with error %d",
-	/*  14 msg_c_myotarget_checkresult             */ "offload error: %s failed with error %d",
-	/*  15 msg_c_offload_descriptor_offload        */ "offload error: cannot find offload entry %s",
-	/*  16 msg_c_merge_var_descs1                  */ "offload error: unexpected number of variable descriptors",
-	/*  17 msg_c_merge_var_descs2                  */ "offload error: unexpected variable type",
-	/*  18 msg_c_mic_parse_env_var_list1           */ "offload_error: MIC environment variable must begin with an alpabetic character",
-	/*  19 msg_c_mic_parse_env_var_list2           */ "offload_error: MIC environment variable value must be specified with ‘=’",
-	/*  20 msg_c_mic_process_exit_ret              */ "offload error: process on the device %d unexpectedly exited with code %d",
-	/*  21 msg_c_mic_process_exit_sig              */ "offload error: process on the device %d was terminated by signal %d (%s)",
-	/*  22 msg_c_mic_process_exit                  */ "offload error: process on the device %d was unexpectedly terminated",
-	/*  23 msg_c_mic_init3                         */ "offload warning: ignoring MIC_STACKSIZE setting; use a value >= 16K and a multiple of 4K",
-	/*  24 msg_c_mic_init4                         */ "offload error: thread key create failed with error %d",
-	/*  25 msg_c_mic_init5                         */ "offload warning: specify OFFLOAD_DEVICES as comma-separated physical device numbers or 'none'",
-	/*  26 msg_c_mic_init6                         */ "offload warning: OFFLOAD_DEVICES device number %d does not correspond to a physical device",
-	/*  27 msg_c_no_static_var_data                */ "offload error: cannot find data associated with statically allocated variable %p",
-	/*  28 msg_c_no_ptr_data                       */ "offload error: cannot find data associated with pointer variable %p",
-	/*  29 msg_c_get_engine_handle                 */ "offload error: cannot get device %d handle (error code %d)",
-	/*  30 msg_c_get_engine_index                  */ "offload error: cannot get physical index for logical device %d (error code %d)",
-	/*  31 msg_c_process_create                    */ "offload error: cannot start process on the device %d (error code %d)",
-	/*  32 msg_c_process_get_func_handles          */ "offload error: cannot get function handles on the device %d (error code %d)",
-	/*  33 msg_c_process_wait_shutdown             */ "offload error: wait for process shutdown failed on device %d (error code %d)",
-	/*  34 msg_c_process_proxy_flush               */ "offload error: cannot flush process output on device %d (error code %d)",
-	/*  35 msg_c_load_library                      */ "offload error: cannot load library to the device %d (error code %d)",
-	/*  36 msg_c_pipeline_create                   */ "offload error: cannot create pipeline on the device %d (error code %d)",
-	/*  37 msg_c_pipeline_run_func                 */ "offload error: cannot execute function on the device %d (error code %d)",
-	/*  38 msg_c_pipeline_start_run_funcs          */ "offload error: cannot start executing pipeline function on the device %d (error code %d)",
-	/*  39 msg_c_buf_create                        */ "offload error: cannot create buffer on device %d (error code %d)",
-	/*  40 msg_c_buf_create_out_of_mem             */ "offload error: cannot create buffer on device %d, out of memory",
-	/*  41 msg_c_buf_create_from_mem               */ "offload error: cannot create buffer from memory on device %d (error code %d)",
-	/*  42 msg_c_buf_destroy                       */ "offload error: buffer destroy failed (error code %d)",
-	/*  43 msg_c_buf_map                           */ "offload error: buffer map failed (error code %d)",
-	/*  44 msg_c_buf_unmap                         */ "offload error: buffer unmap failed (error code %d)",
-	/*  45 msg_c_buf_read                          */ "offload error: buffer read failed (error code %d)",
-	/*  46 msg_c_buf_write                         */ "offload error: buffer write failed (error code %d)",
-	/*  47 msg_c_buf_copy                          */ "offload error: buffer copy failed (error code %d)",
-	/*  48 msg_c_buf_get_address                   */ "offload error: cannot get buffer address on device %d (error code %d)",
-	/*  49 msg_c_buf_add_ref                       */ "offload error: cannot reuse buffer memory on device %d (error code %d)",
-	/*  50 msg_c_buf_release_ref                   */ "offload error: cannot release buffer memory on device %d (error code %d)",
-	/*  51 msg_c_buf_set_state                     */ "offload error: buffer set state failed (error code %d)",
-	/*  52 msg_c_event_wait                        */ "offload error: wait for event to become signaled failed (error code %d)",
-	/*  53 msg_c_zero_or_neg_ptr_len               */ "offload error: memory allocation of negative length is not supported",
-	/*  54 msg_c_zero_or_neg_transfer_size         */ "offload error: data transfer of zero or negative size is not supported",
-	/*  55 msg_c_bad_ptr_mem_range                 */ "offload error: address range partially overlaps with existing allocation",
-	/*  56 msg_c_different_src_and_dstn_sizes      */ "offload error: size of the source %d differs from size of the destination %d",
-	/*  57 msg_c_non_contiguous_dope_vector        */ "offload error: offload data transfer supports only a single contiguous memory range per variable",
-	/*  58 msg_c_omp_invalid_device_num_env        */ "offload warning: ignoring %s setting; use a non-negative integer value",
-	/*  59 msg_c_omp_invalid_device_num            */ "offload error: device number should be a non-negative integer value",
-	/*  60 msg_c_unknown_binary_type               */ "offload error: unexpected embedded target binary type, expected either an executable or shared library",
-	/*  61 msg_c_multiple_target_exes              */ "offload error: more that one target executable found",
-	/*  62 msg_c_no_target_exe                     */ "offload error: target executable is not available",
-	/*  63 msg_c_report_unknown_timer_node         */ "offload error: unknown timer node",
-	/*  64 msg_c_report_unknown_trace_node         */ "offload error: unknown trace node",
-	/*  65 msg_c_report_host                       */ "HOST",
-	/*  66 msg_c_report_mic                        */ "MIC",
-	/*  67 msg_c_report_title                      */ "timer data       (sec)",
-	/*  68 msg_c_report_seconds                    */ "(seconds)",
-	/*  69 msg_c_report_bytes                      */ "(bytes)",
-	/*  70 msg_c_report_cpu_time                   */ "CPU Time",
-	/*  71 msg_c_report_mic_time                   */ "MIC Time",
-	/*  72 msg_c_report_tag                        */ "Tag",
-	/*  73 msg_c_report_from_file                  */ "Offload from file",
-	/*  74 msg_c_report_file                       */ "File",
-	/*  75 msg_c_report_line                       */ "Line",
-	/*  76 msg_c_report_cpu_to_mic_data            */ "CPU->MIC Data",
-	/*  77 msg_c_report_mic_to_cpu_data            */ "MIC->CPU Data",
-	/*  78 msg_c_report_offload                    */ "Offload",
-	/*  79 msg_c_report_w_tag                      */ "Tag %d",
-	/*  80 msg_c_report_state                      */ "State",
-	/*  81 msg_c_report_start                      */ "Start target",
-	/*  82 msg_c_report_init                       */ "Initialize",
-	/*  83 msg_c_report_logical_card               */ "logical card",
-	/*  84 msg_c_report_physical_card              */ "physical card",
-	/*  85 msg_c_report_register                   */ "Register static data tables",
-	/*  86 msg_c_report_init_func                  */ "Setup target entry",
-	/*  87 msg_c_report_create_buf_host            */ "Create host buffer",
-	/*  88 msg_c_report_create_buf_mic             */ "Create target buffer",
-	/*  89 msg_c_report_send_pointer_data          */ "Send pointer data",
-	/*  90 msg_c_report_sent_pointer_data          */ "Host->target pointer data",
-	/*  91 msg_c_report_gather_copyin_data         */ "Gather copyin data",
-	/*  92 msg_c_report_copyin_data                */ "Host->target copyin data",
-	/*  93 msg_c_report_state_signal               */ "Signal",
-	/*  94 msg_c_report_signal                     */ "signal :",
-	/*  95 msg_c_report_wait                       */ "waits  :",
-	/*  96 msg_c_report_compute                    */ "Execute task on target",
-	/*  97 msg_c_report_receive_pointer_data       */ "Receive pointer data",
-	/*  98 msg_c_report_received_pointer_data      */ "Target->host pointer data",
-	/*  99 msg_c_report_start_target_func          */ "Start target entry",
-	/* 100 msg_c_report_var                        */ "Var",
-	/* 101 msg_c_report_scatter_copyin_data        */ "Scatter copyin data",
-	/* 102 msg_c_report_gather_copyout_data        */ "Gather copyout data",
-	/* 103 msg_c_report_scatter_copyout_data       */ "Scatter copyout data",
-	/* 104 msg_c_report_copyout_data               */ "Target->host copyout data",
-	/* 105 msg_c_report_unregister                 */ "Unregister data tables",
-	/* 106 msg_c_report_destroy                    */ "Destroy",
-	/* 107 msg_c_report_myoinit                    */ "Initialize MYO",
-	/* 108 msg_c_report_myoregister                */ "Register MYO tables",
-	/* 109 msg_c_report_myofini                    */ "Finalize MYO",
-	/* 110 msg_c_report_mic_myo_shared             */ "MIC MYO shared table register",
-	/* 111 msg_c_report_mic_myo_fptr               */ "MIC MYO fptr table register",
-	/* 112 msg_c_report_myosharedmalloc            */ "MYO shared malloc",
-	/* 113 msg_c_report_myosharedfree              */ "MYO shared free",
-	/* 114 msg_c_report_myosharedalignedmalloc     */ "MYO shared aligned malloc",
-	/* 115 msg_c_report_myosharedalignedfree       */ "MYO shared aligned free",
-	/* 116 msg_c_report_myoacquire                 */ "MYO acquire",
-	/* 117 msg_c_report_myorelease                 */ "MYO release",
-	/* 118 msg_c_report_host_total_offload_time    */ "host: total offload time",
-	/* 119 msg_c_report_host_initialize            */ "host: initialize target",
-	/* 120 msg_c_report_host_target_acquire        */ "host: acquire target",
-	/* 121 msg_c_report_host_wait_deps             */ "host: wait dependencies",
-	/* 122 msg_c_report_host_setup_buffers         */ "host: setup buffers",
-	/* 123 msg_c_report_host_alloc_buffers         */ "host: allocate buffers",
-	/* 124 msg_c_report_host_setup_misc_data       */ "host: setup misc_data",
-	/* 125 msg_c_report_host_alloc_data_buffer     */ "host: allocate buffer",
-	/* 126 msg_c_report_host_send_pointers         */ "host: send pointers",
-	/* 127 msg_c_report_host_gather_inputs         */ "host: gather inputs",
-	/* 128 msg_c_report_host_map_in_data_buffer    */ "host: map IN data buffer",
-	/* 129 msg_c_report_host_unmap_in_data_buffer  */ "host: unmap IN data buffer",
-	/* 130 msg_c_report_host_start_compute         */ "host: initiate compute",
-	/* 131 msg_c_report_host_wait_compute          */ "host: wait compute",
-	/* 132 msg_c_report_host_start_buffers_reads   */ "host: initiate pointer reads",
-	/* 133 msg_c_report_host_scatter_outputs       */ "host: scatter outputs",
-	/* 134 msg_c_report_host_map_out_data_buffer   */ "host: map OUT data buffer",
-	/* 135 msg_c_report_host_unmap_out_data_buffer */ "host: unmap OUT data buffer",
-	/* 136 msg_c_report_host_wait_buffers_reads    */ "host: wait pointer reads",
-	/* 137 msg_c_report_host_destroy_buffers       */ "host: destroy buffers",
-	/* 138 msg_c_report_target_total_time          */ "target: total time",
-	/* 139 msg_c_report_target_descriptor_setup    */ "target: setup offload descriptor",
-	/* 140 msg_c_report_target_func_lookup         */ "target: entry lookup",
-	/* 141 msg_c_report_target_func_time           */ "target: entry time",
-	/* 142 msg_c_report_target_scatter_inputs      */ "target: scatter inputs",
-	/* 143 msg_c_report_target_add_buffer_refs     */ "target: add buffer reference",
-	/* 144 msg_c_report_target_compute             */ "target: compute",
-	/* 145 msg_c_report_target_gather_outputs      */ "target: gather outputs",
-	/* 146 msg_c_report_target_release_buffer_refs */ "target: remove buffer reference",
-	/* 147 msg_c_coi_pipeline_max_number           */ "number of host threads doing offload exceeds maximum of %d",
-	/* 148 msg_c_ranges_dont_match                 */ "ranges of source and destination don't match together",
-	/* 149 msg_c_destination_is_over               */ "insufficient destination memory to transfer source",
-	/* 150 msg_c_slice_of_noncont_array            */ "a non-contiguous slice may be taken of contiguous arrays only",
-	/* 151 msg_c_pointer_array_mismatch            */ "number of %s elements is less than described by the source",
-};

offload/src/mic_lib.f90

@@ -1,441 +0,0 @@
-!
-!//===----------------------------------------------------------------------===//
-!//
-!//                     The LLVM Compiler Infrastructure
-!//
-!// This file is dual licensed under the MIT and the University of Illinois Open
-!// Source Licenses. See LICENSE.txt for details.
-!//
-!//===----------------------------------------------------------------------===//
-!
-
-
-! **********************************************************************************
-! * This file is intended to support the Intel(r) Many Integrated Core Architecture.
-! **********************************************************************************
-! free form Fortran source - should be named .f90
-! lines are longer than 72 characters
-
-module mic_lib
-use, intrinsic :: iso_c_binding
-
-integer, parameter:: target_mic=2
-integer, parameter:: default_target_type=target_mic
-integer, parameter:: default_target_number=0
-
-enum, bind(C)
-    enumerator :: OFFLOAD_SUCCESS  = 0
-    enumerator :: OFFLOAD_DISABLED          ! offload is disabled
-    enumerator :: OFFLOAD_UNAVAILABLE       ! card is not available
-    enumerator :: OFFLOAD_OUT_OF_MEMORY     ! not enough memory on device
-    enumerator :: OFFLOAD_PROCESS_DIED      ! target process has died
-    enumerator :: OFFLOAD_ERROR             ! unspecified error
-end enum
-
-type, bind (C) :: offload_status
-    integer(kind=c_int)    ::  result          = OFFLOAD_DISABLED
-    integer(kind=c_int)    ::  device_number   = -1
-    integer(kind=c_size_t) ::  data_sent       = 0
-    integer(kind=c_size_t) ::  data_received   = 0
-end type offload_status
-
-interface
-function offload_number_of_devices ()                                  &
-           bind (C, name = "_Offload_number_of_devices")
-!dec$ attributes default :: offload_number_of_devices
-  import :: c_int
-  integer (kind=c_int)        :: offload_number_of_devices
-!dec$ attributes offload:mic :: offload_number_of_devices
-!dir$ attributes known_intrinsic ::  offload_number_of_devices
-end function offload_number_of_devices
-
-function offload_signaled(target_number, signal)                       &
-           bind (C, name = "_Offload_signaled")
-!dec$ attributes default :: offload_signaled
-  import :: c_int, c_int64_t
-  integer (kind=c_int) :: offload_signaled
-  integer (kind=c_int), value :: target_number
-  integer (kind=c_int64_t), value :: signal
-!dec$ attributes offload:mic :: offload_signaled
-end function offload_signaled
-
-subroutine offload_report(val)                                         &
-           bind (C, name = "_Offload_report")
-!dec$ attributes default :: offload_report
-  import :: c_int
-  integer (kind=c_int), value :: val
-!dec$ attributes offload:mic :: offload_report
-end subroutine offload_report
-
-function offload_get_device_number()                                   &
-           bind (C, name = "_Offload_get_device_number")
-!dec$ attributes default :: offload_get_device_number
-  import :: c_int
-  integer (kind=c_int)        :: offload_get_device_number
-!dec$ attributes offload:mic :: offload_get_device_number
-end function offload_get_device_number
-
-function offload_get_physical_device_number()                          &
-           bind (C, name = "_Offload_get_physical_device_number")
-!dec$ attributes default :: offload_get_physical_device_number
-  import :: c_int
-  integer (kind=c_int)        :: offload_get_physical_device_number
-!dec$ attributes offload:mic :: offload_get_physical_device_number
-end function offload_get_physical_device_number
-
-! OpenMP API wrappers
-
-subroutine omp_set_num_threads_target (target_type,                    &
-                                       target_number,                  &
-                                       num_threads)                    &
-           bind (C, name = "omp_set_num_threads_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, num_threads
-end subroutine omp_set_num_threads_target
-
-function omp_get_max_threads_target (target_type,                      &
-                                     target_number)                    &
-         bind (C, name = "omp_get_max_threads_target")
-  import :: c_int
-  integer (kind=c_int)        :: omp_get_max_threads_target
-  integer (kind=c_int), value :: target_type, target_number
-end function omp_get_max_threads_target
-
-function omp_get_num_procs_target (target_type,                        &
-                                   target_number)                      &
-         bind (C, name = "omp_get_num_procs_target")
-  import :: c_int
-  integer (kind=c_int)        :: omp_get_num_procs_target
-  integer (kind=c_int), value :: target_type, target_number
-end function omp_get_num_procs_target
-
-subroutine omp_set_dynamic_target (target_type,                        &
-                                   target_number,                      &
-                                   num_threads)                        &
-           bind (C, name = "omp_set_dynamic_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, num_threads
-end subroutine omp_set_dynamic_target
-
-function omp_get_dynamic_target (target_type,                          &
-                                 target_number)                        &
-         bind (C, name = "omp_get_dynamic_target")
-  import :: c_int
-  integer (kind=c_int)        :: omp_get_dynamic_target
-  integer (kind=c_int), value :: target_type, target_number
-end function omp_get_dynamic_target
-
-subroutine omp_set_nested_target (target_type,                         &
-                                  target_number,                       &
-                                  nested)                              &
-           bind (C, name = "omp_set_nested_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, nested
-end subroutine omp_set_nested_target
-
-function omp_get_nested_target (target_type,                           &
-                                target_number)                         &
-         bind (C, name = "omp_get_nested_target")
-  import :: c_int
-  integer (kind=c_int)        :: omp_get_nested_target
-  integer (kind=c_int), value :: target_type, target_number
-end function omp_get_nested_target
-
-subroutine omp_set_schedule_target (target_type,                       &
-                                    target_number,                     &
-                                    kind,                              &
-                                    modifier)                          &
-           bind (C, name = "omp_set_schedule_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, kind, modifier
-end subroutine omp_set_schedule_target
-
-subroutine omp_get_schedule_target (target_type,                       &
-                                    target_number,                     &
-                                    kind,                              &
-                                    modifier)                          &
-           bind (C, name = "omp_get_schedule_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: kind, modifier
-end subroutine omp_get_schedule_target
-
-! lock API functions
-
-subroutine omp_init_lock_target (target_type,                          &
-                                 target_number,                        &
-                                 lock)                                 &
-           bind (C, name = "omp_init_lock_target")
-  import :: c_int, c_intptr_t
-  !dir$ attributes known_intrinsic ::  omp_init_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_init_lock_target
-
-subroutine omp_destroy_lock_target (target_type,                       &
-                                    target_number,                     &
-                                    lock)                              &
-           bind (C, name = "omp_destroy_lock_target")
-  import :: c_int, c_intptr_t
-  !dir$ attributes known_intrinsic ::  omp_destroy_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_destroy_lock_target
-
-subroutine omp_set_lock_target (target_type,                           &
-                                target_number,                         &
-                                lock)                                  &
-           bind (C, name = "omp_set_lock_target")
-  import :: c_int, c_intptr_t
-  !dir$ attributes known_intrinsic ::  omp_set_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_set_lock_target
-
-subroutine omp_unset_lock_target (target_type,                         &
-                                  target_number,                       &
-                                  lock)                                &
-           bind (C, name = "omp_unset_lock_target")
-  import :: c_int, c_intptr_t
-  !dir$ attributes known_intrinsic ::  omp_unset_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_unset_lock_target
-
-function omp_test_lock_target (target_type,                            &
-                               target_number,                          &
-                               lock)                                   &
-           bind (C, name = "omp_test_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: omp_test_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end function omp_test_lock_target
-
-! nested lock API functions
-
-subroutine omp_init_nest_lock_target (target_type,                     &
-                                      target_number,                   &
-                                      lock)                            &
-           bind (C, name = "omp_init_nest_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_init_nest_lock_target
-
-subroutine omp_destroy_nest_lock_target (target_type,                  &
-                                         target_number,                &
-                                         lock)                         &
-           bind (C, name = "omp_destroy_nest_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_destroy_nest_lock_target
-
-subroutine omp_set_nest_lock_target (target_type,                      &
-                                     target_number,                    &
-                                     lock)                             &
-           bind (C, name = "omp_set_nest_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_set_nest_lock_target
-
-subroutine omp_unset_nest_lock_target (target_type,                    &
-                                       target_number,                  &
-                                       lock)                           &
-           bind (C, name = "omp_unset_nest_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end subroutine omp_unset_nest_lock_target
-
-function omp_test_nest_lock_target (target_type,                       &
-                                    target_number,                     &
-                                    lock)                              &
-           bind (C, name = "omp_test_nest_lock_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: omp_test_nest_lock_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: lock
-end function omp_test_nest_lock_target
-
-! kmp API functions
-
-subroutine kmp_set_stacksize_target (target_type,                      &
-                                     target_number,                    &
-                                     size)                             &
-           bind (C, name = "kmp_set_stacksize_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, size
-end subroutine kmp_set_stacksize_target
-
-function kmp_get_stacksize_target (target_type,                        &
-                                   target_number)                      &
-         bind (C, name = "kmp_get_stacksize_target")
-  import :: c_int
-  integer (kind=c_int)        :: kmp_get_stacksize_target
-  integer (kind=c_int), value :: target_type, target_number
-end function kmp_get_stacksize_target
-
-subroutine kmp_set_stacksize_s_target (target_type,                    &
-                                       target_number,                  &
-                                       size)                           &
-           bind (C, name = "kmp_set_stacksize_s_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, size
-end subroutine kmp_set_stacksize_s_target
-
-function kmp_get_stacksize_s_target (target_type,                      &
-                                     target_number)                    &
-         bind (C, name = "kmp_get_stacksize_s_target")
-  import :: c_int
-  integer (kind=c_int)        :: kmp_get_stacksize_s_target
-  integer (kind=c_int), value :: target_type, target_number
-end function kmp_get_stacksize_s_target
-
-subroutine kmp_set_blocktime_target (target_type,                      &
-                                     target_number,                    &
-                                     time)                             &
-           bind (C, name = "kmp_set_blocktime_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, time
-end subroutine kmp_set_blocktime_target
-
-function kmp_get_blocktime_target (target_type,                        &
-                                   target_number)                      &
-         bind (C, name = "kmp_get_blocktime_target")
-  import :: c_int
-  integer (kind=c_int)        :: kmp_get_blocktime_target
-  integer (kind=c_int), value :: target_type, target_number
-end function kmp_get_blocktime_target
-
-subroutine kmp_set_library_serial_target (target_type,                 &
-                                          target_number)               &
-           bind (C, name = "kmp_set_library_serial_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number
-end subroutine kmp_set_library_serial_target
-
-subroutine kmp_set_library_turnaround_target (target_type,             &
-                                              target_number)           &
-           bind (C, name = "kmp_set_library_turnaround_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number
-end subroutine kmp_set_library_turnaround_target
-
-subroutine kmp_set_library_throughput_target (target_type,             &
-                                              target_number)           &
-           bind (C, name = "kmp_set_library_throughput_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number
-end subroutine kmp_set_library_throughput_target
-
-subroutine kmp_set_library_target (target_type,                        &
-                                   target_number,                      &
-                                   mode)                               &
-           bind (C, name = "kmp_set_library_target")
-  import :: c_int
-  integer (kind=c_int), value :: target_type, target_number, mode
-end subroutine kmp_set_library_target
-
-function kmp_get_library_target (target_type,                          &
-                                 target_number)                        &
-         bind (C, name = "kmp_get_library_target")
-  import :: c_int
-  integer (kind=c_int)        :: kmp_get_library_target
-  integer (kind=c_int), value :: target_type, target_number
-end function kmp_get_library_target
-
-subroutine kmp_set_defaults_target (target_type,                       &
-                                    target_number,                     &
-                                    defaults)                          &
-           bind (C, name = "kmp_set_defaults_target")
-  import :: c_int, c_char
- character (kind=c_char) :: defaults(*)
- integer (kind=c_int), value :: target_type, target_number
-end subroutine kmp_set_defaults_target
-
-! affinity API functions
-
-subroutine kmp_create_affinity_mask_target (target_type,               &
-                                            target_number,             &
-                                            mask)                      &
-           bind (C, name = "kmp_create_affinity_mask_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: mask
-end subroutine kmp_create_affinity_mask_target
-
-subroutine kmp_destroy_affinity_mask_target (target_type,              &
-                                             target_number,            &
-                                             mask)                     &
-           bind (C, name = "kmp_destroy_affinity_mask_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: mask
-end subroutine kmp_destroy_affinity_mask_target
-
-function kmp_set_affinity_target (target_type,                         &
-                                  target_number,                       &
-                                  mask)                                &
-           bind (C, name = "kmp_set_affinity_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: kmp_set_affinity_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: mask
-end function kmp_set_affinity_target
-
-function kmp_get_affinity_target (target_type,                         &
-                                  target_number,                       &
-                                  mask)                                &
-           bind (C, name = "kmp_get_affinity_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: kmp_get_affinity_target
-  integer (kind=c_int), value :: target_type, target_number
-  integer (kind=c_intptr_t), value :: mask
-end function kmp_get_affinity_target
-
-function kmp_get_affinity_max_proc_target (target_type,                &
-                                           target_number)              &
-           bind (C, name = "kmp_get_affinity_max_proc_target")
-  import :: c_int
-  integer (kind=c_int)        :: kmp_get_affinity_max_proc_target
-  integer (kind=c_int), value :: target_type, target_number
-end function kmp_get_affinity_max_proc_target
-
-function kmp_set_affinity_mask_proc_target (target_type,               &
-                                            target_number,             &
-                                            proc,                      &
-                                            mask)                      &
-           bind (C, name = "kmp_set_affinity_mask_proc_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: kmp_set_affinity_mask_proc_target
-  integer (kind=c_int), value :: target_type, target_number, proc
-  integer (kind=c_intptr_t), value :: mask
-end function kmp_set_affinity_mask_proc_target
-
-function kmp_unset_affinity_mask_proc_target (target_type,             &
-                                              target_number,           &
-                                              proc,                    &
-                                              mask)                    &
-           bind (C, name = "kmp_unset_affinity_mask_proc_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: kmp_unset_affinity_mask_proc_target
-  integer (kind=c_int), value :: target_type, target_number, proc
-  integer (kind=c_intptr_t), value :: mask
-end function kmp_unset_affinity_mask_proc_target
-
-function kmp_get_affinity_mask_proc_target (target_type,               &
-                                            target_number,             &
-                                            proc,                      &
-                                            mask)                      &
-           bind (C, name = "kmp_get_affinity_mask_proc_target")
-  import :: c_int, c_intptr_t
-  integer (kind=c_int)        :: kmp_get_affinity_mask_proc_target
-  integer (kind=c_int), value :: target_type, target_number, proc
-  integer (kind=c_intptr_t), value :: mask
-end function kmp_get_affinity_mask_proc_target
-
-end interface
-end module mic_lib

offload/src/offload.h

@@ -1,474 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*
- * Include file for Offload API.
- */
-
-#ifndef OFFLOAD_H_INCLUDED
-#define OFFLOAD_H_INCLUDED
-
-#if defined(LINUX) || defined(FREEBSD)
-#include <bits/functexcept.h>
-#endif
-
-#include <stddef.h>
-#include <omp.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TARGET_ATTRIBUTE __declspec(target(mic))
-
-/*
- *  The target architecture.
- */
-typedef enum TARGET_TYPE {
-    TARGET_NONE,    /* Undefine target */
-    TARGET_HOST,    /* Host used as target */
-    TARGET_MIC      /* MIC target */
-} TARGET_TYPE;
-
-/*
- *  The default target type.
- */
-#define DEFAULT_TARGET_TYPE TARGET_MIC
-
-/*
- *  The default target number.
- */
-#define DEFAULT_TARGET_NUMBER 0
-
-/*
- *  Offload status.
- */
-typedef enum {
-    OFFLOAD_SUCCESS = 0,
-    OFFLOAD_DISABLED,               /* offload is disabled */
-    OFFLOAD_UNAVAILABLE,            /* card is not available */
-    OFFLOAD_OUT_OF_MEMORY,          /* not enough memory on device */
-    OFFLOAD_PROCESS_DIED,           /* target process has died */
-    OFFLOAD_ERROR                   /* unspecified error */
-} _Offload_result;
-
-typedef struct {
-    _Offload_result result;         /* result, see above */
-    int             device_number;  /* device number */
-    size_t          data_sent;      /* number of bytes sent to the target */
-    size_t          data_received;  /* number of bytes received by host */
-} _Offload_status;
-
-#define OFFLOAD_STATUS_INIT(x) \
-    ((x).result = OFFLOAD_DISABLED)
-
-#define OFFLOAD_STATUS_INITIALIZER \
-    { OFFLOAD_DISABLED, -1, 0, 0 }
-
-/* Offload runtime interfaces */
-
-extern int _Offload_number_of_devices(void);
-extern int _Offload_get_device_number(void);
-extern int _Offload_get_physical_device_number(void);
-
-extern void* _Offload_shared_malloc(size_t size);
-extern void  _Offload_shared_free(void *ptr);
-
-extern void* _Offload_shared_aligned_malloc(size_t size, size_t align);
-extern void  _Offload_shared_aligned_free(void *ptr);
-
-extern int _Offload_signaled(int index, void *signal);
-extern void _Offload_report(int val);
-
-/* OpenMP API */
-
-extern void omp_set_default_device(int num);
-extern int  omp_get_default_device(void);
-extern int  omp_get_num_devices(void);
-
-/* OpenMP API wrappers */
-
-/* Set num_threads on target */
-extern void omp_set_num_threads_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-);
-
-/* Get max_threads from target */
-extern int omp_get_max_threads_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-/* Get num_procs from target */
-extern int omp_get_num_procs_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-/* Set dynamic on target */
-extern void omp_set_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-);
-
-/* Get dynamic from target */
-extern int omp_get_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-/* Set nested on target */
-extern void omp_set_nested_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int nested
-);
-
-/* Get nested from target */
-extern int omp_get_nested_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void omp_set_num_threads_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-);
-
-extern int omp_get_max_threads_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern int omp_get_num_procs_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void omp_set_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-);
-
-extern int omp_get_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void omp_set_nested_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-);
-
-extern int omp_get_nested_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void omp_set_schedule_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_sched_t kind,
-    int modifier
-);
-
-extern void omp_get_schedule_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_sched_t *kind,
-    int *modifier
-);
-
-/* lock API functions */
-
-typedef struct {
-    omp_lock_t lock;
-} omp_lock_target_t;
-
-extern void omp_init_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-);
-
-extern void omp_destroy_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-);
-
-extern void omp_set_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-);
-
-extern void omp_unset_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-);
-
-extern int omp_test_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-);
-
-/* nested lock API functions */
-
-typedef struct {
-    omp_nest_lock_t lock;
-} omp_nest_lock_target_t;
-
-extern void omp_init_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-);
-
-extern void omp_destroy_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-);
-
-extern void omp_set_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-);
-
-extern void omp_unset_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-);
-
-extern int omp_test_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-);
-
-/* kmp API functions */
-
-extern void kmp_set_stacksize_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int size
-);
-
-extern int kmp_get_stacksize_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_stacksize_s_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    size_t size
-);
-
-extern size_t kmp_get_stacksize_s_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_blocktime_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int time
-);
-
-extern int kmp_get_blocktime_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_library_serial_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_library_turnaround_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_library_throughput_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_library_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int mode
-);
-
-extern int kmp_get_library_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern void kmp_set_defaults_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    char const *defaults
-);
-
-/* affinity API functions */
-
-typedef struct {
-    kmp_affinity_mask_t mask;
-} kmp_affinity_mask_target_t;
-
-extern void kmp_create_affinity_mask_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern void kmp_destroy_affinity_mask_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern int kmp_set_affinity_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern int kmp_get_affinity_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern int kmp_get_affinity_max_proc_target(
-    TARGET_TYPE target_type,
-    int target_number
-);
-
-extern int kmp_set_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern int kmp_unset_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-);
-
-extern int kmp_get_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-);
-
-#ifdef __cplusplus
-} /* extern "C" */
-
-/* Namespace for the shared_allocator. */
-namespace __offload {
-  /* This follows the specification for std::allocator. */
-  /* Forward declaration of the class template. */
-  template <typename T>
-  class shared_allocator;
-
-  /* Specialization for shared_allocator<void>. */
-  template <>
-  class shared_allocator<void> {
-  public:
-    typedef void       *pointer;
-    typedef const void *const_pointer;
-    typedef void        value_type;
-    template <class U> struct rebind { typedef shared_allocator<U> other; };
-  };
-
-  /* Definition of shared_allocator<T>. */
-  template <class T>
-  class shared_allocator {
-  public:
-    typedef size_t     size_type;
-    typedef ptrdiff_t  difference_type;
-    typedef T         *pointer;
-    typedef const T   *const_pointer;
-    typedef T         &reference;
-    typedef const T   &const_reference;
-    typedef T          value_type;
-    template <class U> struct rebind { typedef shared_allocator<U> other; };
-    shared_allocator() throw() { }
-    shared_allocator(const shared_allocator&) throw() { }
-    template <class U> shared_allocator(const shared_allocator<U>&) throw() { }
-    ~shared_allocator() throw() { }
-    pointer address(reference x) const { return &x; }
-    const_pointer address(const_reference x) const { return &x; }
-    pointer allocate(
-      size_type, shared_allocator<void>::const_pointer hint = 0);
-    void deallocate(pointer p, size_type n);
-    size_type max_size() const throw() {
-      return size_type(-1)/sizeof(T);
-    } /* max_size */
-    void construct(pointer p, const T& arg) {
-      ::new (p) T(arg);
-    } /* construct */
-    void destroy(pointer p) {
-      p->~T();
-    } /* destroy */
-  };
-
-  /* Definition for allocate. */
-  template <class T>
-  typename shared_allocator<T>::pointer
-  shared_allocator<T>::allocate(shared_allocator<T>::size_type s,
-                                shared_allocator<void>::const_pointer) {
-    /* Allocate from shared memory. */
-    void *ptr = _Offload_shared_malloc(s*sizeof(T));
-    if (ptr == 0) std::__throw_bad_alloc();
-    return static_cast<pointer>(ptr);
-  } /* allocate */
-
-  template <class T>
-  void shared_allocator<T>::deallocate(pointer p,
-                                       shared_allocator<T>::size_type) {
-    /* Free the shared memory. */
-    _Offload_shared_free(p);
-  } /* deallocate */
-
-  template <typename _T1, typename _T2>
-  inline bool operator==(const shared_allocator<_T1> &, 
-                         const shared_allocator<_T2> &) throw() {
-    return true;
-  }  /* operator== */
-
-  template <typename _T1, typename _T2>
-  inline bool operator!=(const shared_allocator<_T1> &, 
-                         const shared_allocator<_T2> &) throw() {
-    return false;
-  }  /* operator!= */
-}  /* __offload */
-#endif /* __cplusplus */
-
-#endif /* OFFLOAD_H_INCLUDED */

offload/src/offload_common.cpp

@@ -1,170 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#if defined(LINUX) || defined(FREEBSD)
-#include <mm_malloc.h>
-#endif
-
-#include "offload_common.h"
-
-// The debug routines
-
-#if OFFLOAD_DEBUG > 0
-
-void __dump_bytes(
-    int trace_level,
-    const void *data,
-    int len
-)
-{
-    if (console_enabled > trace_level) {
-        const uint8_t *arr = (const uint8_t*) data;
-        char buffer[4096];
-        char *bufferp;
-        int count = 0;
-
-        bufferp = buffer;
-        while (len--) {
-            sprintf(bufferp, "%02x", *arr++);
-            bufferp += 2;
-            count++;
-            if ((count&3) == 0) {
-                sprintf(bufferp, " ");
-                bufferp++;
-            }
-            if ((count&63) == 0) {
-                OFFLOAD_DEBUG_TRACE(trace_level, "%s\n", buffer);
-                bufferp = buffer;
-                count = 0;
-            }
-        }
-        if (count) {
-            OFFLOAD_DEBUG_TRACE(trace_level, "%s\n", buffer);
-        }
-    }
-}
-#endif // OFFLOAD_DEBUG
-
-// The Marshaller and associated routines
-
-void Marshaller::send_data(
-    const void *data,
-    int64_t length
-)
-{
-    OFFLOAD_DEBUG_TRACE(2, "send_data(%p, %lld)\n",
-                        data, length);
-    memcpy(buffer_ptr, data, (size_t)length);
-    buffer_ptr += length;
-    tfr_size += length;
-}
-
-void Marshaller::receive_data(
-    void *data,
-    int64_t length
-)
-{
-    OFFLOAD_DEBUG_TRACE(2, "receive_data(%p, %lld)\n",
-                        data, length);
-    memcpy(data, buffer_ptr, (size_t)length);
-    buffer_ptr += length;
-    tfr_size += length;
-}
-
-// Send function pointer
-void Marshaller::send_func_ptr(
-    const void* data
-)
-{
-    const char* name;
-    size_t      length;
-
-    if (data != 0) {
-        name = __offload_funcs.find_name(data);
-        if (name == 0) {
-#if OFFLOAD_DEBUG > 0
-            if (console_enabled > 2) {
-                __offload_funcs.dump();
-            }
-#endif // OFFLOAD_DEBUG > 0
-
-            LIBOFFLOAD_ERROR(c_send_func_ptr, data);
-            exit(1);
-        }
-        length = strlen(name) + 1;
-    }
-    else {
-        name = "";
-        length = 1;
-    }
-
-    memcpy(buffer_ptr, name, length);
-    buffer_ptr += length;
-    tfr_size += length;
-}
-
-// Receive function pointer
-void Marshaller::receive_func_ptr(
-    const void** data
-)
-{
-    const char* name;
-    size_t      length;
-
-    name = (const char*) buffer_ptr;
-    if (name[0] != '\0') {
-        *data = __offload_funcs.find_addr(name);
-        if (*data == 0) {
-#if OFFLOAD_DEBUG > 0
-            if (console_enabled > 2) {
-                __offload_funcs.dump();
-            }
-#endif // OFFLOAD_DEBUG > 0
-
-            LIBOFFLOAD_ERROR(c_receive_func_ptr, name);
-            exit(1);
-        }
-        length = strlen(name) + 1;
-    }
-    else {
-        *data = 0;
-        length = 1;
-    }
-
-    buffer_ptr += length;
-    tfr_size += length;
-}
-
-// End of the Marshaller and associated routines
-
-extern void *OFFLOAD_MALLOC(
-    size_t size,
-    size_t align
-)
-{
-    void *ptr;
-    int   err;
-
-    OFFLOAD_DEBUG_TRACE(2, "%s(%lld, %lld)\n", __func__, size, align);
-
-    if (align < sizeof(void*)) {
-        align = sizeof(void*);
-    }
-
-    ptr = _mm_malloc(size, align);
-    if (ptr == NULL) {
-        LIBOFFLOAD_ERROR(c_offload_malloc, size, align);
-        exit(1);
-    }
-
-    OFFLOAD_DEBUG_TRACE(2, "%s returned %p\n", __func__, ptr);
-
-    return ptr;
-}

offload/src/offload_common.h

@@ -1,444 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*! \file
-    \brief The parts of the runtime library common to host and target
-*/
-
-#ifndef OFFLOAD_COMMON_H_INCLUDED
-#define OFFLOAD_COMMON_H_INCLUDED
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <memory.h>
-
-#include "offload.h"
-#include "offload_table.h"
-#include "offload_trace.h"
-#include "offload_timer.h"
-#include "offload_util.h"
-#include "cean_util.h"
-#include "dv_util.h"
-#include "liboffload_error_codes.h"
-
-#include <stdarg.h>
-
-// The debug routines
-
-// Host console and file logging
-extern int console_enabled;
-extern int offload_report_level;
-
-#define OFFLOAD_DO_TRACE (offload_report_level == 3)
-
-extern const char *prefix;
-extern int offload_number;
-#if !HOST_LIBRARY
-extern int mic_index;
-#endif
-
-#if HOST_LIBRARY
-void Offload_Report_Prolog(OffloadHostTimerData* timer_data);
-void Offload_Report_Epilog(OffloadHostTimerData* timer_data);
-void offload_report_free_data(OffloadHostTimerData * timer_data);
-void Offload_Timer_Print(void);
-
-#ifndef TARGET_WINNT
-#define OFFLOAD_DEBUG_INCR_OFLD_NUM() \
-        __sync_add_and_fetch(&offload_number, 1)
-#else
-#define OFFLOAD_DEBUG_INCR_OFLD_NUM() \
-        _InterlockedIncrement(reinterpret_cast<long*>(&offload_number))
-#endif
-
-#define OFFLOAD_DEBUG_PRINT_TAG_PREFIX() \
-        printf("%s:  ", prefix);
-
-#define OFFLOAD_DEBUG_PRINT_PREFIX() \
-        printf("%s:  ", prefix);
-#else
-#define OFFLOAD_DEBUG_PRINT_PREFIX() \
-        printf("%s%d:  ", prefix, mic_index);
-#endif // HOST_LIBRARY
-
-#define OFFLOAD_TRACE(trace_level, ...)  \
-    if (console_enabled >= trace_level) { \
-        OFFLOAD_DEBUG_PRINT_PREFIX(); \
-        printf(__VA_ARGS__); \
-        fflush(NULL); \
-    }
-
-#if OFFLOAD_DEBUG > 0
-
-#define OFFLOAD_DEBUG_TRACE(level, ...) \
-    OFFLOAD_TRACE(level, __VA_ARGS__)
-
-#define OFFLOAD_REPORT(level, offload_number, stage, ...) \
-    if (OFFLOAD_DO_TRACE) { \
-        offload_stage_print(stage, offload_number, __VA_ARGS__); \
-        fflush(NULL); \
-    }
-
-#define OFFLOAD_DEBUG_TRACE_1(level, offload_number, stage, ...) \
-    if (OFFLOAD_DO_TRACE) { \
-        offload_stage_print(stage, offload_number, __VA_ARGS__); \
-        fflush(NULL); \
-    } \
-    if (!OFFLOAD_DO_TRACE) { \
-        OFFLOAD_TRACE(level, __VA_ARGS__) \
-    }
-
-#define OFFLOAD_DEBUG_DUMP_BYTES(level, a, b) \
-    __dump_bytes(level, a, b)
-
-extern void __dump_bytes(
-    int level,
-    const void *data,
-    int len
-);
-
-#else
-
-#define OFFLOAD_DEBUG_LOG(level, ...)
-#define OFFLOAD_DEBUG_DUMP_BYTES(level, a, b)
-
-#endif
-
-// Runtime interface
-
-#define OFFLOAD_PREFIX(a) __offload_##a
-
-#define OFFLOAD_MALLOC            OFFLOAD_PREFIX(malloc)
-#define OFFLOAD_FREE(a)           _mm_free(a)
-
-// Forward functions
-
-extern void *OFFLOAD_MALLOC(size_t size, size_t align);
-
-// The Marshaller
-
-//! \enum Indicator for the type of entry on an offload item list.
-enum OffloadItemType {
-    c_data =   1,       //!< Plain data
-    c_data_ptr,         //!< Pointer data
-    c_func_ptr,         //!< Function pointer
-    c_void_ptr,         //!< void*
-    c_string_ptr,       //!< C string
-    c_dv,               //!< Dope vector variable
-    c_dv_data,          //!< Dope-vector data
-    c_dv_data_slice,    //!< Dope-vector data's slice
-    c_dv_ptr,           //!< Dope-vector variable pointer
-    c_dv_ptr_data,      //!< Dope-vector pointer data
-    c_dv_ptr_data_slice,//!< Dope-vector pointer data's slice
-    c_cean_var,         //!< CEAN variable
-    c_cean_var_ptr,     //!< Pointer to CEAN variable
-    c_data_ptr_array,   //!< Pointer to data pointer array
-    c_func_ptr_array,   //!< Pointer to function pointer array
-    c_void_ptr_array,   //!< Pointer to void* pointer array
-    c_string_ptr_array  //!< Pointer to char* pointer array
-};
-
-#define VAR_TYPE_IS_PTR(t) ((t) == c_string_ptr || \
-                            (t) == c_data_ptr || \
-                            (t) == c_cean_var_ptr || \
-                            (t) == c_dv_ptr)
-
-#define VAR_TYPE_IS_SCALAR(t) ((t) == c_data || \
-                               (t) == c_void_ptr || \
-                               (t) == c_cean_var || \
-                               (t) == c_dv)
-
-#define VAR_TYPE_IS_DV_DATA(t) ((t) == c_dv_data || \
-                                (t) == c_dv_ptr_data)
-
-#define VAR_TYPE_IS_DV_DATA_SLICE(t) ((t) == c_dv_data_slice || \
-                                      (t) == c_dv_ptr_data_slice)
-
-
-//! \enum Specify direction to copy offloaded variable.
-enum OffloadParameterType {
-    c_parameter_unknown = -1, //!< Unknown clause
-    c_parameter_nocopy,       //!< Variable listed in "nocopy" clause
-    c_parameter_in,           //!< Variable listed in "in" clause
-    c_parameter_out,          //!< Variable listed in "out" clause
-    c_parameter_inout         //!< Variable listed in "inout" clause
-};
-
-//! An Offload Variable descriptor
-struct VarDesc {
-    //! OffloadItemTypes of source and destination
-    union {
-        struct {
-            uint8_t dst : 4; //!< OffloadItemType of destination
-            uint8_t src : 4; //!< OffloadItemType of source
-        };
-        uint8_t bits;
-    } type;
-
-    //! OffloadParameterType that describes direction of data transfer
-    union {
-        struct {
-            uint8_t in  : 1; //!< Set if IN or INOUT
-            uint8_t out : 1; //!< Set if OUT or INOUT
-        };
-        uint8_t bits;
-    } direction;
-
-    uint8_t alloc_if;        //!< alloc_if modifier value
-    uint8_t free_if;         //!< free_if modifier value
-    uint32_t align;          //!< MIC alignment requested for pointer data
-    //! Not used by compiler; set to 0
-    /*! Used by runtime as offset to data from start of MIC buffer */
-    uint32_t mic_offset;
-    //! Flags describing this variable
-    union {
-        struct {
-            //! source variable has persistent storage
-            uint32_t is_static : 1;
-            //! destination variable has persistent storage
-            uint32_t is_static_dstn : 1;
-            //! has length for c_dv && c_dv_ptr
-            uint32_t has_length : 1;
-            //! persisted local scalar is in stack buffer
-            uint32_t is_stack_buf : 1;
-            //! buffer address is sent in data
-            uint32_t sink_addr : 1;
-            //! alloc displacement is sent in data
-            uint32_t alloc_disp : 1;
-            //! source data is noncontiguous
-            uint32_t is_noncont_src : 1;
-            //! destination data is noncontiguous
-            uint32_t is_noncont_dst : 1;
-        };
-        uint32_t bits;
-    } flags;
-    //! Not used by compiler; set to 0
-    /*! Used by runtime as offset to base from data stored in a buffer */
-    int64_t offset;
-    //! Element byte-size of data to be transferred
-    /*! For dope-vector, the size of the dope-vector      */
-    int64_t size;
-    union {
-        //! Set to 0 for array expressions and dope-vectors
-        /*! Set to 1 for scalars                          */
-        /*! Set to value of length modifier for pointers  */
-        int64_t count;
-        //! Displacement not used by compiler
-        int64_t disp;
-    };
-
-    //! This field not used by OpenMP 4.0
-    /*! The alloc section expression in #pragma offload   */
-    union {
-       void *alloc;
-       int64_t ptr_arr_offset;
-    };
-
-    //! This field not used by OpenMP 4.0
-    /*! The into section expression in #pragma offload    */
-    /*! For c_data_ptr_array this is the into ptr array   */
-    void *into;
-
-    //! For an ordinary variable, address of the variable
-    /*! For c_cean_var (C/C++ array expression),
-        pointer to arr_desc, which is an array descriptor. */
-    /*! For c_data_ptr_array (array of data pointers),
-        pointer to ptr_array_descriptor,
-        which is a descriptor for pointer array transfers. */
-    void *ptr;
-};
-
-//! Auxiliary struct used when -g is enabled that holds variable names
-struct VarDesc2 {
-    const char *sname; //!< Source name
-    const char *dname; //!< Destination name (when "into" is used)
-};
-
-/*! When the OffloadItemType is c_data_ptr_array
-    the ptr field of the main descriptor points to this struct.          */
-/*! The type in VarDesc1 merely says c_cean_data_ptr, but the pointer
-    type can be c_data_ptr, c_func_ptr, c_void_ptr, or c_string_ptr.
-    Therefore the actual pointer type is in the flags field of VarDesc3. */
-/*! If flag_align_is_array/flag_alloc_if_is_array/flag_free_if_is_array
-    is 0 then alignment/alloc_if/free_if are specified in VarDesc1.      */
-/*! If flag_align_is_array/flag_alloc_if_is_array/flag_free_if_is_array
-    is 1 then align_array/alloc_if_array/free_if_array specify
-    the set of alignment/alloc_if/free_if values.                        */
-/*! For the other fields, if neither the scalar nor the array flag
-    is set, then that modifier was not specified. If the bits are set
-    they specify which modifier was set and whether it was a
-    scalar or an array expression.                                       */
-struct VarDesc3
-{
-    void *ptr_array;        //!< Pointer to arr_desc of array of pointers
-    void *align_array;      //!< Scalar value or pointer to arr_desc
-    void *alloc_if_array;   //!< Scalar value or pointer to arr_desc
-    void *free_if_array;    //!< Scalar value or pointer to arr_desc
-    void *extent_start;     //!< Scalar value or pointer to arr_desc
-    void *extent_elements;  //!< Scalar value or pointer to arr_desc
-    void *into_start;       //!< Scalar value or pointer to arr_desc
-    void *into_elements;    //!< Scalar value or pointer to arr_desc
-    void *alloc_start;      //!< Scalar value or pointer to arr_desc
-    void *alloc_elements;   //!< Scalar value or pointer to arr_desc
-    /*! Flags that describe the pointer type and whether each field
-        is a scalar value or an array expression.        */
-    /*! First 6 bits are pointer array element type:
-        c_data_ptr, c_func_ptr, c_void_ptr, c_string_ptr */
-    /*! Then single bits specify:                        */
-    /*!     align_array is an array                      */
-    /*!     alloc_if_array is an array                   */
-    /*!     free_if_array is an array                    */
-    /*!     extent_start is a scalar expression          */
-    /*!     extent_start is an array expression          */
-    /*!     extent_elements is a scalar expression       */
-    /*!     extent_elements is an array expression       */
-    /*!     into_start is a scalar expression            */
-    /*!     into_start is an array expression            */
-    /*!     into_elements is a scalar expression         */
-    /*!     into_elements is an array expression         */
-    /*!     alloc_start is a scalar expression           */
-    /*!     alloc_start is an array expression           */
-    /*!     alloc_elements is a scalar expression        */
-    /*!     alloc_elements is an array expression        */
-    uint32_t array_fields;
-};
-const int flag_align_is_array = 6;
-const int flag_alloc_if_is_array = 7;
-const int flag_free_if_is_array = 8;
-const int flag_extent_start_is_scalar = 9;
-const int flag_extent_start_is_array = 10;
-const int flag_extent_elements_is_scalar = 11;
-const int flag_extent_elements_is_array = 12;
-const int flag_into_start_is_scalar = 13;
-const int flag_into_start_is_array = 14;
-const int flag_into_elements_is_scalar = 15;
-const int flag_into_elements_is_array = 16;
-const int flag_alloc_start_is_scalar = 17;
-const int flag_alloc_start_is_array = 18;
-const int flag_alloc_elements_is_scalar = 19;
-const int flag_alloc_elements_is_array = 20;
-
-// The Marshaller
-class Marshaller
-{
-private:
-    // Start address of buffer
-    char *buffer_start;
-
-    // Current pointer within buffer
-    char *buffer_ptr;
-
-    // Physical size of data sent (including flags)
-    long long buffer_size;
-
-    // User data sent/received
-    long long tfr_size;
-
-public:
-    // Constructor
-    Marshaller() :
-        buffer_start(0), buffer_ptr(0),
-        buffer_size(0), tfr_size(0)
-    {
-    }
-
-    // Return count of user data sent/received
-    long long get_tfr_size() const
-    {
-        return tfr_size;
-    }
-
-    // Return pointer to buffer
-    char *get_buffer_start() const
-    {
-        return buffer_start;
-    }
-
-    // Return current size of data in buffer
-    long long get_buffer_size() const
-    {
-        return buffer_size;
-    }
-
-    // Set buffer pointer
-    void init_buffer(
-        char *d,
-        long long s
-    )
-    {
-        buffer_start = buffer_ptr = d;
-        buffer_size = s;
-    }
-
-    // Send data
-    void send_data(
-        const void *data,
-        int64_t length
-    );
-
-    // Receive data
-    void receive_data(
-        void *data,
-        int64_t length
-    );
-
-    // Send function pointer
-    void send_func_ptr(
-        const void* data
-    );
-
-    // Receive function pointer
-    void receive_func_ptr(
-        const void** data
-    );
-};
-
-// End of the Marshaller
-
-// The offloaded function descriptor.
-// Sent from host to target to specify which function to run.
-// Also, sets console and file tracing levels.
-struct FunctionDescriptor
-{
-    // Input data size.
-    long long in_datalen;
-
-    // Output data size.
-    long long out_datalen;
-
-    // Whether trace is requested on console.
-    // A value of 1 produces only function name and data sent/received.
-    // Values > 1 produce copious trace information.
-    uint8_t console_enabled;
-
-    // Flag controlling timing on the target side.
-    // Values > 0 enable timing on sink.
-    uint8_t timer_enabled;
-
-    int offload_report_level;
-    int offload_number;
-
-    // number of variable descriptors
-    int vars_num;
-
-    // inout data offset if data is passed as misc/return data
-    // otherwise it should be zero.
-    int data_offset;
-
-    // The name of the offloaded function
-    char data[];
-};
-
-// typedef OFFLOAD.
-// Pointer to OffloadDescriptor.
-typedef struct OffloadDescriptor *OFFLOAD;
-
-#endif // OFFLOAD_COMMON_H_INCLUDED

offload/src/offload_engine.cpp

@@ -1,531 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_engine.h"
-#include <signal.h>
-#include <errno.h>
-
-#include <algorithm>
-#include <vector>
-
-#include "offload_host.h"
-#include "offload_table.h"
-
-const char* Engine::m_func_names[Engine::c_funcs_total] =
-{
-    "server_compute",
-#ifdef MYO_SUPPORT
-    "server_myoinit",
-    "server_myofini",
-#endif // MYO_SUPPORT
-    "server_init",
-    "server_var_table_size",
-    "server_var_table_copy"
-};
-
-// Symbolic representation of system signals. Fix for CQ233593
-const char* Engine::c_signal_names[Engine::c_signal_max] =
-{
-    "Unknown SIGNAL",
-    "SIGHUP",    /*  1, Hangup (POSIX).  */
-    "SIGINT",    /*  2, Interrupt (ANSI).  */
-    "SIGQUIT",   /*  3, Quit (POSIX).  */
-    "SIGILL",    /*  4, Illegal instruction (ANSI).  */
-    "SIGTRAP",   /*  5, Trace trap (POSIX).  */
-    "SIGABRT",   /*  6, Abort (ANSI).  */
-    "SIGBUS",    /*  7, BUS error (4.2 BSD).  */
-    "SIGFPE",    /*  8, Floating-point exception (ANSI).  */
-    "SIGKILL",   /*  9, Kill, unblockable (POSIX).  */
-    "SIGUSR1",   /* 10, User-defined signal 1 (POSIX).  */
-    "SIGSEGV",   /* 11, Segmentation violation (ANSI).  */
-    "SIGUSR2",   /* 12, User-defined signal 2 (POSIX).  */
-    "SIGPIPE",   /* 13, Broken pipe (POSIX).  */
-    "SIGALRM",   /* 14, Alarm clock (POSIX).  */
-    "SIGTERM",   /* 15, Termination (ANSI).  */
-    "SIGSTKFLT", /* 16, Stack fault.  */
-    "SIGCHLD",   /* 17, Child status has changed (POSIX).  */
-    "SIGCONT",   /* 18, Continue (POSIX).  */
-    "SIGSTOP",   /* 19, Stop, unblockable (POSIX).  */
-    "SIGTSTP",   /* 20, Keyboard stop (POSIX).  */
-    "SIGTTIN",   /* 21, Background read from tty (POSIX).  */
-    "SIGTTOU",   /* 22, Background write to tty (POSIX).  */
-    "SIGURG",    /* 23, Urgent condition on socket (4.2 BSD).  */
-    "SIGXCPU",   /* 24, CPU limit exceeded (4.2 BSD).  */
-    "SIGXFSZ",   /* 25, File size limit exceeded (4.2 BSD).  */
-    "SIGVTALRM", /* 26, Virtual alarm clock (4.2 BSD).  */
-    "SIGPROF",   /* 27, Profiling alarm clock (4.2 BSD).  */
-    "SIGWINCH",  /* 28, Window size change (4.3 BSD, Sun).  */
-    "SIGIO",     /* 29, I/O now possible (4.2 BSD).  */
-    "SIGPWR",    /* 30, Power failure restart (System V).  */
-    "SIGSYS"     /* 31, Bad system call.  */
-};
-
-void Engine::init(void)
-{
-    if (!m_ready) {
-        mutex_locker_t locker(m_lock);
-
-        if (!m_ready) {
-            // start process if not done yet
-            if (m_process == 0) {
-                init_process();
-            }
-
-            // load penging images
-            load_libraries();
-
-            // and (re)build pointer table
-            init_ptr_data();
-
-            // it is ready now
-            m_ready = true;
-        }
-    }
-}
-
-void Engine::init_process(void)
-{
-    COIENGINE engine;
-    COIRESULT res;
-    const char **environ;
-
-    // create environment for the target process
-    environ = (const char**) mic_env_vars.create_environ_for_card(m_index);
-    if (environ != 0) {
-        for (const char **p = environ; *p != 0; p++) {
-            OFFLOAD_DEBUG_TRACE(3, "Env Var for card %d: %s\n", m_index, *p);
-        }
-    }
-
-    // Create execution context in the specified device
-    OFFLOAD_DEBUG_TRACE(2, "Getting device %d (engine %d) handle\n", m_index,
-                        m_physical_index);
-    res = COI::EngineGetHandle(COI_ISA_KNC, m_physical_index, &engine);
-    check_result(res, c_get_engine_handle, m_index, res);
-
-    // Target executable should be available by the time when we
-    // attempt to initialize the device
-    if (__target_exe == 0) {
-        LIBOFFLOAD_ERROR(c_no_target_exe);
-        exit(1);
-    }
-
-    OFFLOAD_DEBUG_TRACE(2,
-        "Loading target executable \"%s\" from %p, size %lld\n",
-        __target_exe->name, __target_exe->data, __target_exe->size);
-
-    res = COI::ProcessCreateFromMemory(
-        engine,                 // in_Engine
-        __target_exe->name,     // in_pBinaryName
-        __target_exe->data,     // in_pBinaryBuffer
-        __target_exe->size,     // in_BinaryBufferLength,
-        0,                      // in_Argc
-        0,                      // in_ppArgv
-        environ == 0,           // in_DupEnv
-        environ,                // in_ppAdditionalEnv
-        mic_proxy_io,           // in_ProxyActive
-        mic_proxy_fs_root,      // in_ProxyfsRoot
-        mic_buffer_size,        // in_BufferSpace
-        mic_library_path,       // in_LibrarySearchPath
-        __target_exe->origin,   // in_FileOfOrigin
-        __target_exe->offset,   // in_FileOfOriginOffset
-        &m_process              // out_pProcess
-    );
-    check_result(res, c_process_create, m_index, res);
-
-    // get function handles
-    res = COI::ProcessGetFunctionHandles(m_process, c_funcs_total,
-                                         m_func_names, m_funcs);
-    check_result(res, c_process_get_func_handles, m_index, res);
-
-    // initialize device side
-    pid_t pid = init_device();
-
-    // For IDB
-    if (__dbg_is_attached) {
-        // TODO: we have in-memory executable now.
-        // Check with IDB team what should we provide them now?
-        if (strlen(__target_exe->name) < MAX_TARGET_NAME) {
-            strcpy(__dbg_target_exe_name, __target_exe->name);
-        }
-        __dbg_target_so_pid = pid;
-        __dbg_target_id = m_physical_index;
-        __dbg_target_so_loaded();
-    }
-}
-
-void Engine::fini_process(bool verbose)
-{
-    if (m_process != 0) {
-        uint32_t sig;
-        int8_t ret;
-
-        // destroy target process
-        OFFLOAD_DEBUG_TRACE(2, "Destroying process on the device %d\n",
-                            m_index);
-
-        COIRESULT res = COI::ProcessDestroy(m_process, -1, 0, &ret, &sig);
-        m_process = 0;
-
-        if (res == COI_SUCCESS) {
-            OFFLOAD_DEBUG_TRACE(3, "Device process: signal %d, exit code %d\n",
-                                sig, ret);
-            if (verbose) {
-                if (sig != 0) {
-                    LIBOFFLOAD_ERROR(
-                        c_mic_process_exit_sig, m_index, sig,
-                        c_signal_names[sig >= c_signal_max ? 0 : sig]);
-                }
-                else {
-                    LIBOFFLOAD_ERROR(c_mic_process_exit_ret, m_index, ret);
-                }
-            }
-
-            // for idb
-            if (__dbg_is_attached) {
-                __dbg_target_so_unloaded();
-            }
-        }
-        else {
-            if (verbose) {
-                LIBOFFLOAD_ERROR(c_mic_process_exit, m_index);
-            }
-        }
-    }
-}
-
-void Engine::load_libraries()
-{
-    // load libraries collected so far
-    for (TargetImageList::iterator it = m_images.begin();
-         it != m_images.end(); it++) {
-        OFFLOAD_DEBUG_TRACE(2, "Loading library \"%s\" from %p, size %llu\n",
-                            it->name, it->data, it->size);
-
-        // load library to the device
-        COILIBRARY lib;
-        COIRESULT res;
-        res = COI::ProcessLoadLibraryFromMemory(m_process,
-                                                it->data,
-                                                it->size,
-                                                it->name,
-                                                mic_library_path,
-                                                it->origin,
-                                                it->offset,
-                                                COI_LOADLIBRARY_V1_FLAGS,
-                                                &lib);
-
-        if (res != COI_SUCCESS && res != COI_ALREADY_EXISTS) {
-            check_result(res, c_load_library, m_index, res);
-        }
-    }
-    m_images.clear();
-}
-
-static bool target_entry_cmp(
-    const VarList::BufEntry &l,
-    const VarList::BufEntry &r
-)
-{
-    const char *l_name = reinterpret_cast<const char*>(l.name);
-    const char *r_name = reinterpret_cast<const char*>(r.name);
-    return strcmp(l_name, r_name) < 0;
-}
-
-static bool host_entry_cmp(
-    const VarTable::Entry *l,
-    const VarTable::Entry *r
-)
-{
-    return strcmp(l->name, r->name) < 0;
-}
-
-void Engine::init_ptr_data(void)
-{
-    COIRESULT res;
-    COIEVENT event;
-
-    // Prepare table of host entries
-    std::vector<const VarTable::Entry*> host_table(__offload_vars.begin(),
-                                                   __offload_vars.end());
-
-    // no need to do anything further is host table is empty
-    if (host_table.size() <= 0) {
-        return;
-    }
-
-    // Get var table entries from the target.
-    // First we need to get size for the buffer to copy data
-    struct {
-        int64_t nelems;
-        int64_t length;
-    } params;
-
-    res = COI::PipelineRunFunction(get_pipeline(),
-                                   m_funcs[c_func_var_table_size],
-                                   0, 0, 0,
-                                   0, 0,
-                                   0, 0,
-                                   &params, sizeof(params),
-                                   &event);
-    check_result(res, c_pipeline_run_func, m_index, res);
-
-    res = COI::EventWait(1, &event, -1, 1, 0, 0);
-    check_result(res, c_event_wait, res);
-
-    if (params.length == 0) {
-        return;
-    }
-
-    // create buffer for target entries and copy data to host
-    COIBUFFER buffer;
-    res = COI::BufferCreate(params.length, COI_BUFFER_NORMAL, 0, 0, 1,
-                            &m_process, &buffer);
-    check_result(res, c_buf_create, m_index, res);
-
-    COI_ACCESS_FLAGS flags = COI_SINK_WRITE;
-    res = COI::PipelineRunFunction(get_pipeline(),
-                                   m_funcs[c_func_var_table_copy],
-                                   1, &buffer, &flags,
-                                   0, 0,
-                                   &params.nelems, sizeof(params.nelems),
-                                   0, 0,
-                                   &event);
-    check_result(res, c_pipeline_run_func, m_index, res);
-
-    res = COI::EventWait(1, &event, -1, 1, 0, 0);
-    check_result(res, c_event_wait, res);
-
-    // patch names in target data
-    VarList::BufEntry *target_table;
-    COIMAPINSTANCE map_inst;
-    res = COI::BufferMap(buffer, 0, params.length, COI_MAP_READ_ONLY, 0, 0,
-                         0, &map_inst,
-                         reinterpret_cast<void**>(&target_table));
-    check_result(res, c_buf_map, res);
-
-    VarList::table_patch_names(target_table, params.nelems);
-
-    // and sort entries
-    std::sort(target_table, target_table + params.nelems, target_entry_cmp);
-    std::sort(host_table.begin(), host_table.end(), host_entry_cmp);
-
-    // merge host and target entries and enter matching vars map
-    std::vector<const VarTable::Entry*>::const_iterator hi =
-        host_table.begin();
-    std::vector<const VarTable::Entry*>::const_iterator he =
-        host_table.end();
-    const VarList::BufEntry *ti = target_table;
-    const VarList::BufEntry *te = target_table + params.nelems;
-
-    while (hi != he && ti != te) {
-        int res = strcmp((*hi)->name, reinterpret_cast<const char*>(ti->name));
-        if (res == 0) {
-            // add matching entry to var map
-            std::pair<PtrSet::iterator, bool> res =
-                m_ptr_set.insert(PtrData((*hi)->addr, (*hi)->size));
-
-            // store address for new entries
-            if (res.second) {
-                PtrData *ptr = const_cast<PtrData*>(res.first.operator->());
-                ptr->mic_addr = ti->addr;
-                ptr->is_static = true;
-            }
-
-            hi++;
-            ti++;
-        }
-        else if (res < 0) {
-            hi++;
-        }
-        else {
-            ti++;
-        }
-    }
-
-    // cleanup
-    res = COI::BufferUnmap(map_inst, 0, 0, 0);
-    check_result(res, c_buf_unmap, res);
-
-    res = COI::BufferDestroy(buffer);
-    check_result(res, c_buf_destroy, res);
-}
-
-COIRESULT Engine::compute(
-    const std::list<COIBUFFER> &buffers,
-    const void*         data,
-    uint16_t            data_size,
-    void*               ret,
-    uint16_t            ret_size,
-    uint32_t            num_deps,
-    const COIEVENT*     deps,
-    COIEVENT*           event
-) /* const */
-{
-    COIBUFFER *bufs;
-    COI_ACCESS_FLAGS *flags;
-    COIRESULT res;
-
-    // convert buffers list to array
-    int num_bufs = buffers.size();
-    if (num_bufs > 0) {
-        bufs = (COIBUFFER*) alloca(num_bufs * sizeof(COIBUFFER));
-        flags = (COI_ACCESS_FLAGS*) alloca(num_bufs *
-                                           sizeof(COI_ACCESS_FLAGS));
-
-        int i = 0;
-        for (std::list<COIBUFFER>::const_iterator it = buffers.begin();
-             it != buffers.end(); it++) {
-            bufs[i] = *it;
-
-            // TODO: this should be fixed
-            flags[i++] = COI_SINK_WRITE;
-        }
-    }
-    else {
-        bufs = 0;
-        flags = 0;
-    }
-
-    // start computation
-    res = COI::PipelineRunFunction(get_pipeline(),
-                                   m_funcs[c_func_compute],
-                                   num_bufs, bufs, flags,
-                                   num_deps, deps,
-                                   data, data_size,
-                                   ret, ret_size,
-                                   event);
-    return res;
-}
-
-pid_t Engine::init_device(void)
-{
-    struct init_data {
-        int  device_index;
-        int  devices_total;
-        int  console_level;
-        int  offload_report_level;
-    } data;
-    COIRESULT res;
-    COIEVENT event;
-    pid_t pid;
-
-    OFFLOAD_DEBUG_TRACE_1(2, 0, c_offload_init,
-                          "Initializing device with logical index %d "
-                          "and physical index %d\n",
-                           m_index, m_physical_index);
-
-    // setup misc data
-    data.device_index = m_index;
-    data.devices_total = mic_engines_total;
-    data.console_level = console_enabled;
-    data.offload_report_level = offload_report_level;
-
-    res = COI::PipelineRunFunction(get_pipeline(),
-                                   m_funcs[c_func_init],
-                                   0, 0, 0, 0, 0,
-                                   &data, sizeof(data),
-                                   &pid, sizeof(pid),
-                                   &event);
-    check_result(res, c_pipeline_run_func, m_index, res);
-
-    res = COI::EventWait(1, &event, -1, 1, 0, 0);
-    check_result(res, c_event_wait, res);
-
-    OFFLOAD_DEBUG_TRACE(2, "Device process pid is %d\n", pid);
-
-    return pid;
-}
-
-// data associated with each thread
-struct Thread {
-    Thread(long* addr_coipipe_counter) {
-        m_addr_coipipe_counter = addr_coipipe_counter;
-        memset(m_pipelines, 0, sizeof(m_pipelines));
-    }
-
-    ~Thread() {
-#ifndef TARGET_WINNT
-        __sync_sub_and_fetch(m_addr_coipipe_counter, 1);
-#else // TARGET_WINNT
-        _InterlockedDecrement(m_addr_coipipe_counter);
-#endif // TARGET_WINNT
-        for (int i = 0; i < mic_engines_total; i++) {
-            if (m_pipelines[i] != 0) {
-                COI::PipelineDestroy(m_pipelines[i]);
-            }
-        }
-    }
-
-    COIPIPELINE get_pipeline(int index) const {
-        return m_pipelines[index];
-    }
-
-    void set_pipeline(int index, COIPIPELINE pipeline) {
-        m_pipelines[index] = pipeline;
-    }
-
-    AutoSet& get_auto_vars() {
-        return m_auto_vars;
-    }
-
-private:
-    long*       m_addr_coipipe_counter;
-    AutoSet     m_auto_vars;
-    COIPIPELINE m_pipelines[MIC_ENGINES_MAX];
-};
-
-COIPIPELINE Engine::get_pipeline(void)
-{
-    Thread* thread = (Thread*) thread_getspecific(mic_thread_key);
-    if (thread == 0) {
-        thread = new Thread(&m_proc_number);
-        thread_setspecific(mic_thread_key, thread);
-    }
-
-    COIPIPELINE pipeline = thread->get_pipeline(m_index);
-    if (pipeline == 0) {
-        COIRESULT res;
-        int proc_num;
-
-#ifndef TARGET_WINNT
-        proc_num = __sync_fetch_and_add(&m_proc_number, 1);
-#else // TARGET_WINNT
-        proc_num = _InterlockedIncrement(&m_proc_number);
-#endif // TARGET_WINNT
-
-        if (proc_num > COI_PIPELINE_MAX_PIPELINES) {
-            LIBOFFLOAD_ERROR(c_coipipe_max_number, COI_PIPELINE_MAX_PIPELINES);
-            LIBOFFLOAD_ABORT;
-        }
-        // create pipeline for this thread
-        res = COI::PipelineCreate(m_process, 0, mic_stack_size, &pipeline);
-        check_result(res, c_pipeline_create, m_index, res);
-
-        thread->set_pipeline(m_index, pipeline);
-    }
-    return pipeline;
-}
-
-AutoSet& Engine::get_auto_vars(void)
-{
-    Thread* thread = (Thread*) thread_getspecific(mic_thread_key);
-    if (thread == 0) {
-        thread = new Thread(&m_proc_number);
-        thread_setspecific(mic_thread_key, thread);
-    }
-
-    return thread->get_auto_vars();
-}
-
-void Engine::destroy_thread_data(void *data)
-{
-    delete static_cast<Thread*>(data);
-}

offload/src/offload_engine.h

@@ -1,482 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef OFFLOAD_ENGINE_H_INCLUDED
-#define OFFLOAD_ENGINE_H_INCLUDED
-
-#include <limits.h>
-
-#include <list>
-#include <set>
-#include <map>
-#include "offload_common.h"
-#include "coi/coi_client.h"
-
-// Address range
-class MemRange {
-public:
-    MemRange() : m_start(0), m_length(0) {}
-    MemRange(const void *addr, uint64_t len) : m_start(addr), m_length(len) {}
-
-    const void* start() const {
-        return m_start;
-    }
-
-    const void* end() const {
-        return static_cast<const char*>(m_start) + m_length;
-    }
-
-    uint64_t length() const {
-        return m_length;
-    }
-
-    // returns true if given range overlaps with another one
-    bool overlaps(const MemRange &o) const {
-        // Two address ranges A[start, end) and B[start,end) overlap
-        // if A.start < B.end and A.end > B.start.
-        return start() < o.end() && end() > o.start();
-    }
-
-    // returns true if given range contains the other range
-    bool contains(const MemRange &o) const {
-        return start() <= o.start() && o.end() <= end();
-    }
-
-private:
-    const void* m_start;
-    uint64_t    m_length;
-};
-
-// Data associated with a pointer variable
-class PtrData {
-public:
-    PtrData(const void *addr, uint64_t len) :
-        cpu_addr(addr, len), cpu_buf(0),
-        mic_addr(0), alloc_disp(0), mic_buf(0), mic_offset(0),
-        ref_count(0), is_static(false)
-    {}
-
-    //
-    // Copy constructor
-    //
-    PtrData(const PtrData& ptr):
-        cpu_addr(ptr.cpu_addr), cpu_buf(ptr.cpu_buf),
-        mic_addr(ptr.mic_addr), alloc_disp(ptr.alloc_disp),
-        mic_buf(ptr.mic_buf), mic_offset(ptr.mic_offset),
-        ref_count(ptr.ref_count), is_static(ptr.is_static)
-    {}
-
-    bool operator<(const PtrData &o) const {
-        // Variables are sorted by the CPU start address.
-        // Overlapping memory ranges are considered equal.
-        return (cpu_addr.start() < o.cpu_addr.start()) &&
-               !cpu_addr.overlaps(o.cpu_addr);
-    }
-
-    long add_reference() {
-        if (is_static) {
-            return LONG_MAX;
-        }
-#ifndef TARGET_WINNT
-        return __sync_fetch_and_add(&ref_count, 1);
-#else // TARGET_WINNT
-        return _InterlockedIncrement(&ref_count) - 1;
-#endif // TARGET_WINNT
-    }
-
-    long remove_reference() {
-        if (is_static) {
-            return LONG_MAX;
-        }
-#ifndef TARGET_WINNT
-        return __sync_sub_and_fetch(&ref_count, 1);
-#else // TARGET_WINNT
-        return _InterlockedDecrement(&ref_count);
-#endif // TARGET_WINNT
-    }
-
-    long get_reference() const {
-        if (is_static) {
-            return LONG_MAX;
-        }
-        return ref_count;
-    }
-
-public:
-    // CPU address range
-    const MemRange  cpu_addr;
-
-    // CPU and MIC buffers
-    COIBUFFER       cpu_buf;
-    COIBUFFER       mic_buf;
-
-    // placeholder for buffer address on mic
-    uint64_t        mic_addr;
-
-    uint64_t        alloc_disp;
-
-    // additional offset to pointer data on MIC for improving bandwidth for
-    // data which is not 4K aligned
-    uint32_t        mic_offset;
-
-    // if true buffers are created from static memory
-    bool            is_static;
-    mutex_t         alloc_ptr_data_lock;
-
-private:
-    // reference count for the entry
-    long            ref_count;
-};
-
-typedef std::list<PtrData*> PtrDataList;
-
-// Data associated with automatic variable
-class AutoData {
-public:
-    AutoData(const void *addr, uint64_t len) :
-        cpu_addr(addr, len), ref_count(0)
-    {}
-
-    bool operator<(const AutoData &o) const {
-        // Variables are sorted by the CPU start address.
-        // Overlapping memory ranges are considered equal.
-        return (cpu_addr.start() < o.cpu_addr.start()) &&
-               !cpu_addr.overlaps(o.cpu_addr);
-    }
-
-    long add_reference() {
-#ifndef TARGET_WINNT
-        return __sync_fetch_and_add(&ref_count, 1);
-#else // TARGET_WINNT
-        return _InterlockedIncrement(&ref_count) - 1;
-#endif // TARGET_WINNT
-    }
-
-    long remove_reference() {
-#ifndef TARGET_WINNT
-        return __sync_sub_and_fetch(&ref_count, 1);
-#else // TARGET_WINNT
-        return _InterlockedDecrement(&ref_count);
-#endif // TARGET_WINNT
-    }
-
-    long get_reference() const {
-        return ref_count;
-    }
-
-public:
-    // CPU address range
-    const MemRange cpu_addr;
-
-private:
-    // reference count for the entry
-    long ref_count;
-};
-
-// Set of autimatic variables
-typedef std::set<AutoData> AutoSet;
-
-// Target image data
-struct TargetImage
-{
-    TargetImage(const char *_name, const void *_data, uint64_t _size,
-                const char *_origin, uint64_t _offset) :
-        name(_name), data(_data), size(_size),
-        origin(_origin), offset(_offset)
-    {}
-
-    // library name
-    const char* name;
-
-    // contents and size
-    const void* data;
-    uint64_t    size;
-
-    // file of origin and offset within that file
-    const char* origin;
-    uint64_t    offset;
-};
-
-typedef std::list<TargetImage> TargetImageList;
-
-// Data associated with persistent auto objects
-struct PersistData
-{
-    PersistData(const void *addr, uint64_t routine_num, uint64_t size) :
-        stack_cpu_addr(addr), routine_id(routine_num)
-    {
-        stack_ptr_data = new PtrData(0, size);
-    }
-    // 1-st key value - beginning of the stack at CPU
-    const void *   stack_cpu_addr;
-    // 2-nd key value - identifier of routine invocation at CPU
-    uint64_t   routine_id;
-    // corresponded PtrData; only stack_ptr_data->mic_buf is used
-    PtrData * stack_ptr_data;
-    // used to get offset of the variable in stack buffer
-    char * cpu_stack_addr;
-};
-
-typedef std::list<PersistData> PersistDataList;
-
-// class representing a single engine
-struct Engine {
-    friend void __offload_init_library_once(void);
-    friend void __offload_fini_library(void);
-
-#define check_result(res, tag, ...) \
-    { \
-        if (res == COI_PROCESS_DIED) { \
-            fini_process(true); \
-            exit(1); \
-        } \
-        if (res != COI_SUCCESS) { \
-            __liboffload_error_support(tag, __VA_ARGS__); \
-            exit(1); \
-        } \
-    }
-
-    int get_logical_index() const {
-        return m_index;
-    }
-
-    int get_physical_index() const {
-        return m_physical_index;
-    }
-
-    const COIPROCESS& get_process() const {
-        return m_process;
-    }
-
-    // initialize device
-    void init(void);
-
-    // add new library
-    void add_lib(const TargetImage &lib)
-    {
-        m_lock.lock();
-        m_ready = false;
-        m_images.push_back(lib);
-        m_lock.unlock();
-    }
-
-    COIRESULT compute(
-        const std::list<COIBUFFER> &buffers,
-        const void*         data,
-        uint16_t            data_size,
-        void*               ret,
-        uint16_t            ret_size,
-        uint32_t            num_deps,
-        const COIEVENT*     deps,
-        COIEVENT*           event
-    );
-
-#ifdef MYO_SUPPORT
-    // temporary workaround for blocking behavior for myoiLibInit/Fini calls
-    void init_myo(COIEVENT *event) {
-        COIRESULT res;
-        res = COI::PipelineRunFunction(get_pipeline(),
-                                       m_funcs[c_func_myo_init],
-                                       0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                       event);
-        check_result(res, c_pipeline_run_func, m_index, res);
-    }
-
-    void fini_myo(COIEVENT *event) {
-        COIRESULT res;
-        res = COI::PipelineRunFunction(get_pipeline(),
-                                       m_funcs[c_func_myo_fini],
-                                       0, 0, 0, 0, 0, 0, 0, 0, 0,
-                                       event);
-        check_result(res, c_pipeline_run_func, m_index, res);
-    }
-#endif // MYO_SUPPORT
-
-    //
-    // Memory association table
-    //
-    PtrData* find_ptr_data(const void *ptr) {
-        m_ptr_lock.lock();
-        PtrSet::iterator res = m_ptr_set.find(PtrData(ptr, 0));
-        m_ptr_lock.unlock();
-        if (res == m_ptr_set.end()) {
-            return 0;
-        }
-        return const_cast<PtrData*>(res.operator->());
-    }
-
-    PtrData* insert_ptr_data(const void *ptr, uint64_t len, bool &is_new) {
-        m_ptr_lock.lock();
-        std::pair<PtrSet::iterator, bool> res =
-            m_ptr_set.insert(PtrData(ptr, len));
-        PtrData* ptr_data = const_cast<PtrData*>(res.first.operator->());
-        m_ptr_lock.unlock();
-
-        is_new = res.second;
-        if (is_new) {
-            // It's necessary to lock as soon as possible.
-            // unlock must be done at call site of insert_ptr_data at
-            // branch for is_new
-            ptr_data->alloc_ptr_data_lock.lock();
-        }
-        return ptr_data;
-    }
-
-    void remove_ptr_data(const void *ptr) {
-        m_ptr_lock.lock();
-        m_ptr_set.erase(PtrData(ptr, 0));
-        m_ptr_lock.unlock();
-    }
-
-    //
-    // Automatic variables
-    //
-    AutoData* find_auto_data(const void *ptr) {
-        AutoSet &auto_vars = get_auto_vars();
-        AutoSet::iterator res = auto_vars.find(AutoData(ptr, 0));
-        if (res == auto_vars.end()) {
-            return 0;
-        }
-        return const_cast<AutoData*>(res.operator->());
-    }
-
-    AutoData* insert_auto_data(const void *ptr, uint64_t len) {
-        AutoSet &auto_vars = get_auto_vars();
-        std::pair<AutoSet::iterator, bool> res =
-            auto_vars.insert(AutoData(ptr, len));
-        return const_cast<AutoData*>(res.first.operator->());
-    }
-
-    void remove_auto_data(const void *ptr) {
-        get_auto_vars().erase(AutoData(ptr, 0));
-    }
-
-    //
-    // Signals
-    //
-    void add_signal(const void *signal, OffloadDescriptor *desc) {
-        m_signal_lock.lock();
-        m_signal_map[signal] = desc;
-        m_signal_lock.unlock();
-    }
-
-    OffloadDescriptor* find_signal(const void *signal, bool remove) {
-        OffloadDescriptor *desc = 0;
-
-        m_signal_lock.lock();
-        {
-            SignalMap::iterator it = m_signal_map.find(signal);
-            if (it != m_signal_map.end()) {
-                desc = it->second;
-                if (remove) {
-                    m_signal_map.erase(it);
-                }
-            }
-        }
-        m_signal_lock.unlock();
-
-        return desc;
-    }
-
-    // stop device process
-    void fini_process(bool verbose);
-
-    // list of stacks active at the engine
-    PersistDataList m_persist_list;
-
-private:
-    Engine() : m_index(-1), m_physical_index(-1), m_process(0), m_ready(false),
-               m_proc_number(0)
-    {}
-
-    ~Engine() {
-        if (m_process != 0) {
-            fini_process(false);
-        }
-    }
-
-    // set indexes
-    void set_indexes(int logical_index, int physical_index) {
-        m_index = logical_index;
-        m_physical_index = physical_index;
-    }
-
-    // start process on device
-    void init_process();
-
-    void load_libraries(void);
-    void init_ptr_data(void);
-
-    // performs library intialization on the device side
-    pid_t init_device(void);
-
-private:
-    // get pipeline associated with a calling thread
-    COIPIPELINE get_pipeline(void);
-
-    // get automatic vars set associated with the calling thread
-    AutoSet& get_auto_vars(void);
-
-    // destructor for thread data
-    static void destroy_thread_data(void *data);
-
-private:
-    typedef std::set<PtrData> PtrSet;
-    typedef std::map<const void*, OffloadDescriptor*> SignalMap;
-
-    // device indexes
-    int         m_index;
-    int         m_physical_index;
-
-    // number of COI pipes created for the engine
-    long        m_proc_number;
-
-    // process handle
-    COIPROCESS  m_process;
-
-    // If false, device either has not been initialized or new libraries
-    // have been added.
-    bool        m_ready;
-    mutex_t     m_lock;
-
-    // List of libraries to be loaded
-    TargetImageList m_images;
-
-    // var table
-    PtrSet      m_ptr_set;
-    mutex_t     m_ptr_lock;
-
-    // signals
-    SignalMap m_signal_map;
-    mutex_t   m_signal_lock;
-
-    // constants for accessing device function handles
-    enum {
-        c_func_compute = 0,
-#ifdef MYO_SUPPORT
-        c_func_myo_init,
-        c_func_myo_fini,
-#endif // MYO_SUPPORT
-        c_func_init,
-        c_func_var_table_size,
-        c_func_var_table_copy,
-        c_funcs_total
-    };
-    static const char* m_func_names[c_funcs_total];
-
-    // device function handles
-    COIFUNCTION m_funcs[c_funcs_total];
-
-    // int -> name mapping for device signals
-    static const int   c_signal_max = 32;
-    static const char* c_signal_names[c_signal_max];
-};
-
-#endif // OFFLOAD_ENGINE_H_INCLUDED

offload/src/offload_env.cpp

@@ -1,354 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_env.h"
-#include <string.h>
-#include <ctype.h>
-#include "offload_util.h"
-#include "liboffload_error_codes.h"
-
-// for environment variables valid on all cards
-const int MicEnvVar::any_card = -1;
-
-MicEnvVar::~MicEnvVar()
-{
-    for (std::list<MicEnvVar::CardEnvVars*>::const_iterator
-         it = card_spec_list.begin();
-         it != card_spec_list.end(); it++) {
-        CardEnvVars *card_data = *it;
-        delete card_data;
-    }
-}
-
-MicEnvVar::VarValue::~VarValue()
-{
-    free(env_var_value);
-}
-
-MicEnvVar::CardEnvVars::~CardEnvVars()
-{
-    for (std::list<MicEnvVar::VarValue*>::const_iterator it = env_vars.begin();
-        it != env_vars.end(); it++) {
-            VarValue *var_value = *it;
-            delete var_value;
-    }
-}
-
-// Searching for card in "card_spec_list" list with the same "number"
-
-MicEnvVar::CardEnvVars* MicEnvVar::get_card(int number)
-{
-    if (number == any_card) {
-        return &common_vars;
-    }
-    for (std::list<MicEnvVar::CardEnvVars*>::const_iterator
-         it = card_spec_list.begin();
-         it != card_spec_list.end(); it++) {
-        CardEnvVars *card_data = *it;
-        if (card_data->card_number == number) {
-            return card_data;
-        }
-    }
-    return NULL;
-}
-
-// Searching for environment variable in "env_var" list with the same name
-
-MicEnvVar::VarValue* MicEnvVar::CardEnvVars::find_var(
-    char* env_var_name,
-    int env_var_name_length
-)
-{
-    for (std::list<MicEnvVar::VarValue*>::const_iterator it = env_vars.begin();
-        it != env_vars.end(); it++) {
-            VarValue *var_value = *it;
-            if (var_value->length == env_var_name_length &&
-                !strncmp(var_value->env_var, env_var_name,
-                         env_var_name_length)) {
-                return var_value;
-            }
-    }
-    return NULL;
-}
-
-void MicEnvVar::analyze_env_var(char *env_var_string)
-{
-    char          *env_var_name;
-    char          *env_var_def;
-    int           card_number;
-    int           env_var_name_length;
-    MicEnvVarKind env_var_kind;
-
-    env_var_kind = get_env_var_kind(env_var_string,
-                                    &card_number,
-                                    &env_var_name,
-                                    &env_var_name_length,
-                                    &env_var_def);
-    switch (env_var_kind) {
-        case c_mic_var:
-        case c_mic_card_var:
-            add_env_var(card_number,
-                        env_var_name,
-                        env_var_name_length,
-                        env_var_def);
-            break;
-        case c_mic_card_env:
-            mic_parse_env_var_list(card_number, env_var_def);
-            break;
-        case c_no_mic:
-        default:
-            break;
-    }
-}
-
-void MicEnvVar::add_env_var(
-    int card_number,
-    char *env_var_name,
-    int env_var_name_length,
-    char *env_var_def
-)
-{
-    VarValue *var;
-    CardEnvVars *card;
-
-    // The case corresponds to common env var definition of kind
-    // <mic-prefix>_<var>
-    if (card_number == any_card) {
-        card = &common_vars;
-    }
-    else {
-        card = get_card(card_number);
-        if (!card) {
-            // definition for new card occurred
-            card = new CardEnvVars(card_number);
-            card_spec_list.push_back(card);
-        }
-
-    }
-    var = card->find_var(env_var_name, env_var_name_length);
-    if (!var) {
-        // put new env var definition in "env_var" list
-        var = new VarValue(env_var_name, env_var_name_length, env_var_def);
-        card->env_vars.push_back(var);
-    }
-}
-
-// The routine analyses string pointed by "env_var_string" argument
-// according to the following syntax:
-//
-// Specification of prefix for MIC environment variables
-// MIC_ENV_PREFIX=<mic-prefix>
-//
-// Setting single MIC environment variable
-// <mic-prefix>_<var>=<value>
-// <mic-prefix>_<card-number>_<var>=<value>
-
-// Setting multiple MIC environment variables
-// <mic-prefix>_<card-number>_ENV=<env-vars>
-
-MicEnvVarKind MicEnvVar::get_env_var_kind(
-    char *env_var_string,
-    int *card_number,
-    char **env_var_name,
-    int *env_var_name_length,
-    char **env_var_def
-)
-{
-    int len = strlen(prefix);
-    char *c = env_var_string;
-    int num = 0;
-    bool card_is_set = false;
-
-    if (strncmp(c, prefix, len) != 0 || c[len] != '_') {
-            return c_no_mic;
-    }
-    c += len + 1;
-
-    *card_number = any_card;
-    if (isdigit(*c)) {
-        while (isdigit (*c)) {
-            num = (*c++ - '0') + (num * 10);
-        }
-    if (*c != '_') {
-        return c_no_mic;
-    }
-    c++;
-        *card_number = num;
-        card_is_set = true;
-    }
-    if (!isalpha(*c)) {
-        return c_no_mic;
-    }
-    *env_var_name = *env_var_def = c;
-    if (strncmp(c, "ENV=", 4) == 0) {
-        if (!card_is_set) {
-            *env_var_name_length = 3;
-            *env_var_name = *env_var_def = c;
-            *env_var_def = strdup(*env_var_def);
-            return  c_mic_var;
-        }
-        *env_var_def = c + strlen("ENV=");
-        *env_var_def = strdup(*env_var_def);
-        return c_mic_card_env;
-    }
-    if (isalpha(*c)) {
-        *env_var_name_length = 0;
-        while (isalnum(*c) || *c == '_') {
-            c++;
-            (*env_var_name_length)++;
-        }
-    }
-    if (*c != '=') {
-        return c_no_mic;
-    }
-    *env_var_def = strdup(*env_var_def);
-    return card_is_set? c_mic_card_var : c_mic_var;
-}
-
-// analysing <env-vars> in form:
-// <mic-prefix>_<card-number>_ENV=<env-vars>
-// where:
-//
-// <env-vars>:
-//                <env-var>
-//                <env-vars> | <env-var>
-//
-// <env-var>:
-//                variable=value
-//                variable="value"
-//                variable=
-
-void MicEnvVar::mic_parse_env_var_list(
-    int card_number, char *env_vars_def_list)
-{
-    char *c = env_vars_def_list;
-    char *env_var_name;
-    int  env_var_name_length;
-    char *env_var_def;
-    bool var_is_quoted;
-
-    if (*c == '"') {
-        c++;
-    }
-    while (*c != 0) {
-        var_is_quoted = false;
-        env_var_name = c;
-        env_var_name_length = 0;
-        if (isalpha(*c)) {
-            while (isalnum(*c) || *c == '_') {
-                c++;
-                env_var_name_length++;
-            }
-        }
-        else {
-            LIBOFFLOAD_ERROR(c_mic_parse_env_var_list1);
-            return;
-        }
-        if (*c != '=') {
-            LIBOFFLOAD_ERROR(c_mic_parse_env_var_list2);
-            return;
-        }
-        c++;
-
-        if (*c == '"') {
-            var_is_quoted = true;
-            c++;
-        }
-        // Environment variable values that contain | will need to be escaped.
-        while (*c != 0 && *c != '|' &&
-               (!var_is_quoted || *c != '"'))
-        {
-            // skip escaped symbol
-            if (*c == '\\') {
-                c++;
-            }
-            c++;
-        }
-        if (var_is_quoted) {
-            c++; // for "
-            while (*c != 0 && *c != '|') {
-                c++;
-            }
-        }
-
-        int sz = c - env_var_name;
-        env_var_def = (char*)malloc(sz);
-        memcpy(env_var_def, env_var_name, sz);
-        env_var_def[sz] = 0;
-
-        if (*c == '|') {
-            c++;
-            while (*c != 0 && *c == ' ') {
-                c++;
-            }
-        }
-        add_env_var(card_number,
-                    env_var_name,
-                    env_var_name_length,
-                    env_var_def);
-    }
-}
-
-// Collect all definitions for the card with number "card_num".
-// The returned result is vector of string pointers defining one
-// environment variable. The vector is terminated by NULL pointer.
-// In the beginning of the vector there are env vars defined as
-// <mic-prefix>_<card-number>_<var>=<value>
-// or
-// <mic-prefix>_<card-number>_ENV=<env-vars>
-// where <card-number> is equal to "card_num"
-// They are followed by definitions valid for any card
-// and absent in previous definitions.
-
-char** MicEnvVar::create_environ_for_card(int card_num)
-{
-    VarValue *var_value;
-    VarValue *var_value_find;
-    CardEnvVars *card_data = get_card(card_num);
-    CardEnvVars *card_data_common;
-    std::list<char*> new_env;
-    char **rez;
-
-    if (!prefix) {
-        return NULL;
-    }
-    // There is no personel env var definitions for the card with
-    // number "card_num"
-    if (!card_data) {
-        return create_environ_for_card(any_card);
-    }
-
-    for (std::list<MicEnvVar::VarValue*>::const_iterator
-         it = card_data->env_vars.begin();
-         it != card_data->env_vars.end(); it++) {
-        var_value = *it;
-        new_env.push_back(var_value->env_var_value);
-    }
-
-    if (card_num != any_card) {
-        card_data_common = get_card(any_card);
-        for (std::list<MicEnvVar::VarValue*>::const_iterator
-             it = card_data_common->env_vars.begin();
-             it != card_data_common->env_vars.end(); it++) {
-            var_value = *it;
-            var_value_find = card_data->find_var(var_value->env_var,
-                                                 var_value->length);
-            if (!var_value_find) {
-                new_env.push_back(var_value->env_var_value);
-            }
-        }
-    }
-
-    int new_env_size = new_env.size();
-    rez = (char**) malloc((new_env_size + 1) * sizeof(char*));
-    std::copy(new_env.begin(), new_env.end(), rez);
-    rez[new_env_size] = 0;
-    return rez;
-}

offload/src/offload_env.h

@@ -1,91 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef OFFLOAD_ENV_H_INCLUDED
-#define OFFLOAD_ENV_H_INCLUDED
-
-#include <list>
-
-// data structure and routines to parse MIC user environment and pass to MIC
-
-enum MicEnvVarKind
-{
-    c_no_mic,         // not MIC env var
-    c_mic_var,        // for <mic-prefix>_<var>
-    c_mic_card_var,   // for <mic-prefix>_<card-number>_<var>
-    c_mic_card_env    // for <mic-prefix>_<card-number>_ENV
-};
-
-struct MicEnvVar {
-public:
-    MicEnvVar() : prefix(0) {}
-    ~MicEnvVar();
-
-    void analyze_env_var(char *env_var_string);
-    char** create_environ_for_card(int card_num);
-    MicEnvVarKind get_env_var_kind(
-        char *env_var_string,
-        int *card_number,
-        char **env_var_name,
-        int *env_var_name_length,
-        char **env_var_def
-    );
-    void add_env_var(
-        int card_number,
-        char *env_var_name,
-        int env_var_name_length,
-        char *env_var_def
-    );
-
-    void set_prefix(const char *pref) {
-        prefix = (pref && *pref != '\0') ? pref : 0;
-    }
-
-    struct VarValue {
-    public:
-        char* env_var;
-        int   length;
-        char* env_var_value;
-
-        VarValue(char* var, int ln, char* value)
-        {
-            env_var = var;
-            length = ln;
-            env_var_value = value;
-        }
-        ~VarValue();
-    };
-
-    struct CardEnvVars {
-    public:
-
-        int card_number;
-        std::list<struct VarValue*> env_vars;
-
-        CardEnvVars() { card_number = any_card; }
-        CardEnvVars(int num) { card_number = num; }
-        ~CardEnvVars();
-
-        void add_new_env_var(int number, char *env_var, int length,
-                             char *env_var_value);
-        VarValue* find_var(char* env_var_name, int env_var_name_length);
-    };
-    static const int any_card;
-
-private:
-    void         mic_parse_env_var_list(int card_number, char *env_var_def);
-    CardEnvVars* get_card(int number);
-
-    const char *prefix;
-    std::list<struct CardEnvVars *> card_spec_list;
-    CardEnvVars common_vars;
-};
-
-#endif // OFFLOAD_ENV_H_INCLUDED

offload/src/offload_host.h

@@ -1,343 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*! \file
-    \brief The parts of the runtime library used only on the host
-*/
-
-#ifndef OFFLOAD_HOST_H_INCLUDED
-#define OFFLOAD_HOST_H_INCLUDED
-
-#ifndef TARGET_WINNT
-#include <unistd.h>
-#endif // TARGET_WINNT
-#include "offload_common.h"
-#include "offload_util.h"
-#include "offload_engine.h"
-#include "offload_env.h"
-#include "offload_orsl.h"
-#include "coi/coi_client.h"
-
-// MIC engines.
-extern Engine*  mic_engines;
-extern uint32_t mic_engines_total;
-
-//! The target image is packed as follows.
-/*!      1. 8 bytes containing the size of the target binary          */
-/*!      2. a null-terminated string which is the binary name         */
-/*!      3. <size> number of bytes that are the contents of the image */
-/*!      The address of symbol __offload_target_image
-             is the address of this structure.                        */
-struct Image {
-     int64_t size; //!< Size in bytes of the target binary name and contents
-     char data[];  //!< The name and contents of the target image
-};
-
-// The offload descriptor.
-class OffloadDescriptor
-{
-public:
-    OffloadDescriptor(
-        int index,
-        _Offload_status *status,
-        bool is_mandatory,
-        bool is_openmp,
-        OffloadHostTimerData * timer_data
-    ) :
-        m_device(mic_engines[index % mic_engines_total]),
-        m_is_mandatory(is_mandatory),
-        m_is_openmp(is_openmp),
-        m_inout_buf(0),
-        m_func_desc(0),
-        m_func_desc_size(0),
-        m_in_deps(0),
-        m_in_deps_total(0),
-        m_out_deps(0),
-        m_out_deps_total(0),
-        m_vars(0),
-        m_vars_extra(0),
-        m_status(status),
-        m_timer_data(timer_data)
-    {}
-
-    ~OffloadDescriptor()
-    {
-        if (m_in_deps != 0) {
-            free(m_in_deps);
-        }
-        if (m_out_deps != 0) {
-            free(m_out_deps);
-        }
-        if (m_func_desc != 0) {
-            free(m_func_desc);
-        }
-        if (m_vars != 0) {
-            free(m_vars);
-            free(m_vars_extra);
-        }
-    }
-
-    bool offload(const char *name, bool is_empty,
-                 VarDesc *vars, VarDesc2 *vars2, int vars_total,
-                 const void **waits, int num_waits, const void **signal,
-                 int entry_id, const void *stack_addr);
-    bool offload_finish();
-
-    bool is_signaled();
-
-    OffloadHostTimerData* get_timer_data() const {
-        return m_timer_data;
-    }
-
-private:
-    bool wait_dependencies(const void **waits, int num_waits);
-    bool setup_descriptors(VarDesc *vars, VarDesc2 *vars2, int vars_total,
-                           int entry_id, const void *stack_addr);
-    bool setup_misc_data(const char *name);
-    bool send_pointer_data(bool is_async);
-    bool send_noncontiguous_pointer_data(
-        int i,
-        PtrData* src_buf,
-        PtrData* dst_buf,
-        COIEVENT *event);
-    bool receive_noncontiguous_pointer_data(
-        int i,
-        char* src_data,
-        COIBUFFER dst_buf,
-        COIEVENT *event);
-
-    bool gather_copyin_data();
-
-    bool compute();
-
-    bool receive_pointer_data(bool is_async);
-    bool scatter_copyout_data();
-
-    void cleanup();
-
-    bool find_ptr_data(PtrData* &ptr_data, void *base, int64_t disp,
-                       int64_t length, bool error_does_not_exist = true);
-    bool alloc_ptr_data(PtrData* &ptr_data, void *base, int64_t disp,
-                        int64_t length, int64_t alloc_disp, int align);
-    bool init_static_ptr_data(PtrData *ptr_data);
-    bool init_mic_address(PtrData *ptr_data);
-    bool offload_stack_memory_manager(const void * stack_begin, int routine_id,
-                                      int buf_size, int align, bool *is_new);
-    bool nullify_target_stack(COIBUFFER targ_buf, uint64_t size);
-
-    bool gen_var_descs_for_pointer_array(int i);
-
-    void report_coi_error(error_types msg, COIRESULT res);
-    _Offload_result translate_coi_error(COIRESULT res) const;
-
-private:
-    typedef std::list<COIBUFFER> BufferList;
-
-    // extra data associated with each variable descriptor
-    struct VarExtra {
-        PtrData* src_data;
-        PtrData* dst_data;
-        AutoData* auto_data;
-        int64_t cpu_disp;
-        int64_t cpu_offset;
-        CeanReadRanges *read_rng_src;
-        CeanReadRanges *read_rng_dst;
-        int64_t ptr_arr_offset;
-        bool is_arr_ptr_el;
-    };
-
-    template<typename T> class ReadArrElements {
-    public:
-        ReadArrElements():
-            ranges(NULL),
-            el_size(sizeof(T)),
-            offset(0),
-            count(0),
-            is_empty(true),
-            base(NULL)
-        {}
-
-        bool read_next(bool flag)
-        {
-            if (flag != 0) {
-                if (is_empty) {
-                    if (ranges) {
-                        if (!get_next_range(ranges, &offset)) {
-                            // ranges are over
-                            return false;
-                        }
-                    }
-                    // all contiguous elements are over
-                    else if (count != 0) {
-                        return false;
-                    }
-
-                    length_cur = size;
-                }
-                else {
-                    offset += el_size;
-                }
-                val = (T)get_el_value(base, offset, el_size);
-                length_cur -= el_size;
-                count++;
-                is_empty = length_cur == 0;
-            }
-            return true;
-        }
-    public:
-        CeanReadRanges * ranges;
-        T       val;
-        int     el_size;
-        int64_t size,
-                offset,
-                length_cur;
-        bool    is_empty;
-        int     count;
-        char   *base;
-    };
-
-    // ptr_data for persistent auto objects
-    PtrData*    m_stack_ptr_data;
-    PtrDataList m_destroy_stack;
-
-    // Engine
-    Engine& m_device;
-
-    // if true offload is mandatory
-    bool m_is_mandatory;
-
-    // if true offload has openmp origin
-    const bool m_is_openmp;
-
-    // The Marshaller for the inputs of the offloaded region.
-    Marshaller m_in;
-
-    // The Marshaller for the outputs of the offloaded region.
-    Marshaller m_out;
-
-    // List of buffers that are passed to dispatch call
-    BufferList m_compute_buffers;
-
-    // List of buffers that need to be destroyed at the end of offload
-    BufferList m_destroy_buffers;
-
-    // Variable descriptors
-    VarDesc*  m_vars;
-    VarExtra* m_vars_extra;
-    int       m_vars_total;
-
-    // Pointer to a user-specified status variable
-    _Offload_status *m_status;
-
-    // Function descriptor
-    FunctionDescriptor* m_func_desc;
-    uint32_t            m_func_desc_size;
-
-    // Buffer for transferring copyin/copyout data
-    COIBUFFER m_inout_buf;
-
-    // Dependencies
-    COIEVENT *m_in_deps;
-    uint32_t  m_in_deps_total;
-    COIEVENT *m_out_deps;
-    uint32_t  m_out_deps_total;
-
-    // Timer data
-    OffloadHostTimerData *m_timer_data;
-
-    // copyin/copyout data length
-    uint64_t m_in_datalen;
-    uint64_t m_out_datalen;
-
-    // a boolean value calculated in setup_descriptors. If true we need to do
-    // a run function on the target. Otherwise it may be optimized away.
-    bool m_need_runfunction;
-};
-
-// Initialization types for MIC
-enum OffloadInitType {
-    c_init_on_start,         // all devices before entering main
-    c_init_on_offload,       // single device before starting the first offload
-    c_init_on_offload_all    // all devices before starting the first offload
-};
-
-// Initializes library and registers specified offload image.
-extern "C" void __offload_register_image(const void* image);
-extern "C" void __offload_unregister_image(const void* image);
-
-// Initializes offload runtime library.
-extern int __offload_init_library(void);
-
-// thread data for associating pipelines with threads
-extern pthread_key_t mic_thread_key;
-
-// Environment variables for devices
-extern MicEnvVar mic_env_vars;
-
-// CPU frequency
-extern uint64_t cpu_frequency;
-
-// LD_LIBRARY_PATH for MIC libraries
-extern char* mic_library_path;
-
-// stack size for target
-extern uint32_t mic_stack_size;
-
-// Preallocated memory size for buffers on MIC
-extern uint64_t mic_buffer_size;
-
-// Setting controlling inout proxy
-extern bool  mic_proxy_io;
-extern char* mic_proxy_fs_root;
-
-// Threshold for creating buffers with large pages
-extern uint64_t __offload_use_2mb_buffers;
-
-// offload initialization type
-extern OffloadInitType __offload_init_type;
-
-// Device number to offload to when device is not explicitly specified.
-extern int __omp_device_num;
-
-// target executable
-extern TargetImage* __target_exe;
-
-// IDB support
-
-// Called by the offload runtime after initialization of offload infrastructure
-// has been completed.
-extern "C" void  __dbg_target_so_loaded();
-
-// Called by the offload runtime when the offload infrastructure is about to be
-// shut down, currently at application exit.
-extern "C" void  __dbg_target_so_unloaded();
-
-// Null-terminated string containing path to the process image of the hosting
-// application (offload_main)
-#define MAX_TARGET_NAME 512
-extern "C" char  __dbg_target_exe_name[MAX_TARGET_NAME];
-
-// Integer specifying the process id
-extern "C" pid_t __dbg_target_so_pid;
-
-// Integer specifying the 0-based device number
-extern "C" int   __dbg_target_id;
-
-// Set to non-zero by the host-side debugger to enable offload debugging
-// support
-extern "C" int   __dbg_is_attached;
-
-// Major version of the debugger support API
-extern "C" const int __dbg_api_major_version;
-
-// Minor version of the debugger support API
-extern "C" const int __dbg_api_minor_version;
-
-#endif // OFFLOAD_HOST_H_INCLUDED

offload/src/offload_myo_host.cpp

@@ -1,805 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_myo_host.h"
-#include <errno.h>
-#include <malloc.h>
-#include "offload_host.h"
-
-#if defined(LINUX) || defined(FREEBSD)
-#include <mm_malloc.h>
-#endif
-
-#define MYO_VERSION1    "MYO_1.0"
-
-extern "C" void __cilkrts_cilk_for_32(void*, void*, uint32_t, int32_t);
-extern "C" void __cilkrts_cilk_for_64(void*, void*, uint64_t, int32_t);
-
-#ifndef TARGET_WINNT
-#pragma weak __cilkrts_cilk_for_32
-#pragma weak __cilkrts_cilk_for_64
-#endif // TARGET_WINNT
-
-#ifdef TARGET_WINNT
-#define MYO_TABLE_END_MARKER() reinterpret_cast<const char*>(-1)
-#else // TARGET_WINNT
-#define MYO_TABLE_END_MARKER() reinterpret_cast<const char*>(0)
-#endif // TARGET_WINNT
-
-class MyoWrapper {
-public:
-    MyoWrapper() : m_lib_handle(0), m_is_available(false)
-    {}
-
-    bool is_available() const {
-        return m_is_available;
-    }
-
-    bool LoadLibrary(void);
-
-    // unloads the library
-    void UnloadLibrary(void) {
-//        if (m_lib_handle != 0) {
-//            DL_close(m_lib_handle);
-//            m_lib_handle = 0;
-//        }
-    }
-
-    // Wrappers for MYO client functions
-    void LibInit(void *arg, void *func) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoinit,
-                                 "%s(%p, %p)\n", __func__, arg, func);
-        CheckResult(__func__, m_lib_init(arg, func));
-    }
-
-    void LibFini(void) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myofini, "%s()\n", __func__);
-        m_lib_fini();
-    }
-
-    void* SharedMalloc(size_t size) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedmalloc,
-                                 "%s(%lld)\n", __func__, size);
-        return m_shared_malloc(size);
-    }
-
-    void SharedFree(void *ptr) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedfree,
-                                 "%s(%p)\n", __func__, ptr);
-        m_shared_free(ptr);
-    }
-
-    void* SharedAlignedMalloc(size_t size, size_t align) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedalignedmalloc,
-                                 "%s(%lld, %lld)\n", __func__, size, align);
-        return m_shared_aligned_malloc(size, align);
-    }
-
-    void SharedAlignedFree(void *ptr) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedalignedfree,
-                              "%s(%p)\n", __func__, ptr);
-        m_shared_aligned_free(ptr);
-    }
-
-    void Acquire(void) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoacquire,
-                              "%s()\n", __func__);
-        CheckResult(__func__, m_acquire());
-    }
-
-    void Release(void) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myorelease,
-                            "%s()\n", __func__);
-        CheckResult(__func__, m_release());
-    }
-
-    void HostVarTablePropagate(void *table, int num_entries) const {
-        OFFLOAD_DEBUG_TRACE(4, "%s(%p, %d)\n", __func__, table, num_entries);
-        CheckResult(__func__, m_host_var_table_propagate(table, num_entries));
-    }
-
-    void HostFptrTableRegister(void *table, int num_entries,
-                               int ordered) const {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoregister,
-                            "%s(%p, %d, %d)\n", __func__, table,
-                            num_entries, ordered);
-        CheckResult(__func__,
-                    m_host_fptr_table_register(table, num_entries, ordered));
-    }
-
-    void RemoteThunkCall(void *thunk, void *args, int device) {
-        OFFLOAD_DEBUG_TRACE(4, "%s(%p, %p, %d)\n", __func__, thunk, args,
-                            device);
-        CheckResult(__func__, m_remote_thunk_call(thunk, args, device));
-    }
-
-    MyoiRFuncCallHandle RemoteCall(char *func, void *args, int device) const {
-        OFFLOAD_DEBUG_TRACE(4, "%s(%s, %p, %d)\n", __func__, func, args,
-                            device);
-        return m_remote_call(func, args, device);
-    }
-
-    void GetResult(MyoiRFuncCallHandle handle) const {
-        OFFLOAD_DEBUG_TRACE(4, "%s(%p)\n", __func__, handle);
-        CheckResult(__func__, m_get_result(handle));
-    }
-
-private:
-    void CheckResult(const char *func, MyoError error) const {
-        if (error != MYO_SUCCESS) {
-             LIBOFFLOAD_ERROR(c_myowrapper_checkresult, func, error);
-            exit(1);
-        }
-    }
-
-private:
-    void* m_lib_handle;
-    bool  m_is_available;
-
-    // pointers to functions from myo library
-    MyoError (*m_lib_init)(void*, void*);
-    void     (*m_lib_fini)(void);
-    void*    (*m_shared_malloc)(size_t);
-    void     (*m_shared_free)(void*);
-    void*    (*m_shared_aligned_malloc)(size_t, size_t);
-    void     (*m_shared_aligned_free)(void*);
-    MyoError (*m_acquire)(void);
-    MyoError (*m_release)(void);
-    MyoError (*m_host_var_table_propagate)(void*, int);
-    MyoError (*m_host_fptr_table_register)(void*, int, int);
-    MyoError (*m_remote_thunk_call)(void*, void*, int);
-    MyoiRFuncCallHandle (*m_remote_call)(char*, void*, int);
-    MyoError (*m_get_result)(MyoiRFuncCallHandle);
-};
-
-bool MyoWrapper::LoadLibrary(void)
-{
-#ifndef TARGET_WINNT
-    const char *lib_name = "libmyo-client.so";
-#else // TARGET_WINNT
-    const char *lib_name = "myo-client.dll";
-#endif // TARGET_WINNT
-
-    OFFLOAD_DEBUG_TRACE(2, "Loading MYO library %s ...\n", lib_name);
-
-    m_lib_handle = DL_open(lib_name);
-    if (m_lib_handle == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to load the library. errno = %d\n",
-                            errno);
-        return false;
-    }
-
-    m_lib_init = (MyoError (*)(void*, void*))
-        DL_sym(m_lib_handle, "myoiLibInit", MYO_VERSION1);
-    if (m_lib_init == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiLibInit");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_lib_fini = (void (*)(void))
-        DL_sym(m_lib_handle, "myoiLibFini", MYO_VERSION1);
-    if (m_lib_fini == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiLibFini");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_shared_malloc = (void* (*)(size_t))
-        DL_sym(m_lib_handle, "myoSharedMalloc", MYO_VERSION1);
-    if (m_shared_malloc == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoSharedMalloc");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_shared_free = (void (*)(void*))
-        DL_sym(m_lib_handle, "myoSharedFree", MYO_VERSION1);
-    if (m_shared_free == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoSharedFree");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_shared_aligned_malloc = (void* (*)(size_t, size_t))
-        DL_sym(m_lib_handle, "myoSharedAlignedMalloc", MYO_VERSION1);
-    if (m_shared_aligned_malloc == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoSharedAlignedMalloc");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_shared_aligned_free = (void (*)(void*))
-        DL_sym(m_lib_handle, "myoSharedAlignedFree", MYO_VERSION1);
-    if (m_shared_aligned_free == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoSharedAlignedFree");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_acquire = (MyoError (*)(void))
-        DL_sym(m_lib_handle, "myoAcquire", MYO_VERSION1);
-    if (m_acquire == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoAcquire");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_release = (MyoError (*)(void))
-        DL_sym(m_lib_handle, "myoRelease", MYO_VERSION1);
-    if (m_release == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoRelease");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_host_var_table_propagate = (MyoError (*)(void*, int))
-        DL_sym(m_lib_handle, "myoiHostVarTablePropagate", MYO_VERSION1);
-    if (m_host_var_table_propagate == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiHostVarTablePropagate");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_host_fptr_table_register = (MyoError (*)(void*, int, int))
-        DL_sym(m_lib_handle, "myoiHostFptrTableRegister", MYO_VERSION1);
-    if (m_host_fptr_table_register == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiHostFptrTableRegister");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_remote_thunk_call = (MyoError (*)(void*, void*, int))
-        DL_sym(m_lib_handle, "myoiRemoteThunkCall", MYO_VERSION1);
-    if (m_remote_thunk_call == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiRemoteThunkCall");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_remote_call = (MyoiRFuncCallHandle (*)(char*, void*, int))
-        DL_sym(m_lib_handle, "myoiRemoteCall", MYO_VERSION1);
-    if (m_remote_call == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiRemoteCall");
-        UnloadLibrary();
-        return false;
-    }
-
-    m_get_result = (MyoError (*)(MyoiRFuncCallHandle))
-        DL_sym(m_lib_handle, "myoiGetResult", MYO_VERSION1);
-    if (m_get_result == 0) {
-        OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
-                            "myoiGetResult");
-        UnloadLibrary();
-        return false;
-    }
-
-    OFFLOAD_DEBUG_TRACE(2, "The library was successfully loaded\n");
-
-    m_is_available = true;
-
-    return true;
-}
-
-static bool myo_is_available;
-static MyoWrapper myo_wrapper;
-
-struct MyoTable
-{
-    MyoTable(SharedTableEntry *tab, int len) : var_tab(tab), var_tab_len(len)
-    {}
-
-    SharedTableEntry*   var_tab;
-    int                 var_tab_len;
-};
-
-typedef std::list<MyoTable> MyoTableList;
-static MyoTableList __myo_table_list;
-static mutex_t      __myo_table_lock;
-static bool         __myo_tables = false;
-
-static void __offload_myo_shared_table_register(SharedTableEntry *entry);
-static void __offload_myo_shared_init_table_register(InitTableEntry* entry);
-static void __offload_myo_fptr_table_register(FptrTableEntry *entry);
-
-static void __offload_myoLoadLibrary_once(void)
-{
-    if (__offload_init_library()) {
-        myo_wrapper.LoadLibrary();
-    }
-}
-
-static bool __offload_myoLoadLibrary(void)
-{
-    static OffloadOnceControl ctrl = OFFLOAD_ONCE_CONTROL_INIT;
-    __offload_run_once(&ctrl, __offload_myoLoadLibrary_once);
-
-    return myo_wrapper.is_available();
-}
-
-static void __offload_myoInit_once(void)
-{
-    if (!__offload_myoLoadLibrary()) {
-        return;
-    }
-
-    // initialize all devices
-    for (int i = 0; i < mic_engines_total; i++) {
-        mic_engines[i].init();
-    }
-
-    // load and initialize MYO library
-    OFFLOAD_DEBUG_TRACE(2, "Initializing MYO library ...\n");
-
-    COIEVENT events[MIC_ENGINES_MAX];
-    MyoiUserParams params[MIC_ENGINES_MAX+1];
-
-    // load target library to all devices
-    for (int i = 0; i < mic_engines_total; i++) {
-        mic_engines[i].init_myo(&events[i]);
-
-        params[i].type = MYOI_USERPARAMS_DEVID;
-        params[i].nodeid = mic_engines[i].get_physical_index() + 1;
-    }
-
-    params[mic_engines_total].type = MYOI_USERPARAMS_LAST_MSG;
-
-    // initialize myo runtime on host
-    myo_wrapper.LibInit(params, 0);
-
-    // wait for the target init calls to finish
-    COIRESULT res;
-    res = COI::EventWait(mic_engines_total, events, -1, 1, 0, 0);
-    if (res != COI_SUCCESS) {
-        LIBOFFLOAD_ERROR(c_event_wait, res);
-        exit(1);
-    }
-
-    myo_is_available = true;
-
-    OFFLOAD_DEBUG_TRACE(2, "Initializing MYO library ... done\n");
-}
-
-static bool __offload_myoInit(void)
-{
-    static OffloadOnceControl ctrl = OFFLOAD_ONCE_CONTROL_INIT;
-    __offload_run_once(&ctrl, __offload_myoInit_once);
-
-    // register pending shared var tables
-    if (myo_is_available && __myo_tables) {
-        mutex_locker_t locker(__myo_table_lock);
-
-        if (__myo_tables) {
-            //  Register tables with MYO so it can propagate to target.
-            for(MyoTableList::const_iterator it = __myo_table_list.begin();
-                it != __myo_table_list.end(); ++it) {
-#ifdef TARGET_WINNT
-                for (SharedTableEntry *entry = it->var_tab;
-                     entry->varName != MYO_TABLE_END_MARKER(); entry++) {
-                    if (entry->varName == 0) {
-                        continue;
-                    }
-                    myo_wrapper.HostVarTablePropagate(entry, 1);
-                }
-#else // TARGET_WINNT
-                myo_wrapper.HostVarTablePropagate(it->var_tab,
-                                                  it->var_tab_len);
-#endif // TARGET_WINNT
-            }
-
-            __myo_table_list.clear();
-            __myo_tables = false;
-        }
-    }
-
-    return myo_is_available;
-}
-
-static bool shared_table_entries(
-    SharedTableEntry *entry
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    for (; entry->varName != MYO_TABLE_END_MARKER(); entry++) {
-#ifdef TARGET_WINNT
-        if (entry->varName == 0) {
-            continue;
-        }
-#endif // TARGET_WINNT
-
-        return true;
-    }
-
-    return false;
-}
-
-static bool fptr_table_entries(
-    FptrTableEntry *entry
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
-#ifdef TARGET_WINNT
-        if (entry->funcName == 0) {
-            continue;
-        }
-#endif // TARGET_WINNT
-
-        return true;
-    }
-
-    return false;
-}
-
-extern "C" void __offload_myoRegisterTables(
-    InitTableEntry* init_table,
-    SharedTableEntry *shared_table,
-    FptrTableEntry *fptr_table
-)
-{
-    // check whether we need to initialize MYO library. It is
-    // initialized only if at least one myo table is not empty
-    if (shared_table_entries(shared_table) || fptr_table_entries(fptr_table)) {
-        // make sure myo library is loaded
-        __offload_myoLoadLibrary();
-
-        // register tables
-        __offload_myo_shared_table_register(shared_table);
-        __offload_myo_fptr_table_register(fptr_table);
-        __offload_myo_shared_init_table_register(init_table);
-    }
-}
-
-void __offload_myoFini(void)
-{
-    if (myo_is_available) {
-        OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-        COIEVENT events[MIC_ENGINES_MAX];
-
-        // kick off myoiLibFini calls on all devices
-        for (int i = 0; i < mic_engines_total; i++) {
-            mic_engines[i].fini_myo(&events[i]);
-        }
-
-        // cleanup myo runtime on host
-        myo_wrapper.LibFini();
-
-        // wait for the target fini calls to finish
-        COIRESULT res;
-        res = COI::EventWait(mic_engines_total, events, -1, 1, 0, 0);
-        if (res != COI_SUCCESS) {
-            LIBOFFLOAD_ERROR(c_event_wait, res);
-            exit(1);
-        }
-    }
-}
-
-static void __offload_myo_shared_table_register(
-    SharedTableEntry *entry
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    SharedTableEntry *start = entry;
-    int entries = 0;
-
-    // allocate shared memory for vars
-    for (; entry->varName != MYO_TABLE_END_MARKER(); entry++) {
-#ifdef TARGET_WINNT
-        if (entry->varName == 0) {
-            OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoSharedTable entry\n");
-            continue;
-        }
-#endif // TARGET_WINNT
-
-        OFFLOAD_DEBUG_TRACE(4, "registering MyoSharedTable entry for %s @%p\n",
-                            entry->varName, entry);
-
-        // Invoke the function to create shared memory
-        reinterpret_cast<void(*)(void)>(entry->sharedAddr)();
-        entries++;
-    }
-
-    // and table to the list if it is not empty
-    if (entries > 0) {
-        mutex_locker_t locker(__myo_table_lock);
-        __myo_table_list.push_back(MyoTable(start, entries));
-        __myo_tables = true;
-    }
-}
-
-static void __offload_myo_shared_init_table_register(InitTableEntry* entry)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-#ifdef TARGET_WINNT
-    for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
-        if (entry->funcName == 0) {
-            OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoSharedInit entry\n");
-            continue;
-        }
-
-        //  Invoke the function to init the shared memory
-        entry->func();
-    }
-#else // TARGET_WINNT
-    for (; entry->func != 0; entry++) {
-        // Invoke the function to init the shared memory
-        entry->func();
-    }
-#endif // TARGET_WINNT
-}
-
-static void __offload_myo_fptr_table_register(
-    FptrTableEntry *entry
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    FptrTableEntry *start = entry;
-    int entries = 0;
-
-    for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
-#ifdef TARGET_WINNT
-        if (entry->funcName == 0) {
-            OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoFptrTable entry\n");
-            continue;
-        }
-#endif // TARGET_WINNT
-
-        if (!myo_wrapper.is_available()) {
-            *(static_cast<void**>(entry->localThunkAddr)) = entry->funcAddr;
-        }
-
-        OFFLOAD_DEBUG_TRACE(4, "registering MyoFptrTable entry for %s @%p\n",
-                            entry->funcName, entry);
-
-#ifdef TARGET_WINNT
-        if (myo_wrapper.is_available()) {
-            myo_wrapper.HostFptrTableRegister(entry, 1, false);
-        }
-#endif // TARGET_WINNT
-
-        entries++;
-    }
-
-#ifndef TARGET_WINNT
-    if (myo_wrapper.is_available() && entries > 0) {
-        myo_wrapper.HostFptrTableRegister(start, entries, false);
-    }
-#endif // TARGET_WINNT
-}
-
-extern "C" int __offload_myoIsAvailable(int target_number)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%d)\n", __func__, target_number);
-
-    if (target_number >= -2) {
-        bool is_default_number = (target_number == -2);
-
-        if (__offload_myoInit()) {
-            if (target_number >= 0) {
-                // User provided the device number
-                int num = target_number % mic_engines_total;
-
-                // reserve device in ORSL
-                target_number = ORSL::reserve(num) ? num : -1;
-            }
-            else {
-                // try to use device 0
-                target_number = ORSL::reserve(0) ? 0 : -1;
-            }
-
-            // make sure device is initialized
-            if (target_number >= 0) {
-                mic_engines[target_number].init();
-            }
-        }
-        else {
-            // fallback to CPU
-            target_number = -1;
-        }
-
-        if (target_number < 0 && !is_default_number) {
-            LIBOFFLOAD_ERROR(c_device_is_not_available);
-            exit(1);
-        }
-    }
-    else {
-        LIBOFFLOAD_ERROR(c_invalid_device_number);
-        exit(1);
-    }
-
-    return target_number;
-}
-
-extern "C" void __offload_myoiRemoteIThunkCall(
-    void *thunk,
-    void *arg,
-    int target_number
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p, %p, %d)\n", __func__, thunk, arg,
-                        target_number);
-
-    myo_wrapper.Release();
-    myo_wrapper.RemoteThunkCall(thunk, arg, target_number);
-    myo_wrapper.Acquire();
-
-    ORSL::release(target_number);
-}
-
-extern "C" void* _Offload_shared_malloc(size_t size)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%lld)\n", __func__, size);
-
-    if (__offload_myoLoadLibrary()) {
-        return myo_wrapper.SharedMalloc(size);
-    }
-    else {
-        return malloc(size);
-    }
-}
-
-extern "C" void _Offload_shared_free(void *ptr)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
-
-    if (__offload_myoLoadLibrary()) {
-        myo_wrapper.SharedFree(ptr);
-    }
-    else {
-        free(ptr);
-    }
-}
-
-extern "C" void* _Offload_shared_aligned_malloc(size_t size, size_t align)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%lld, %lld)\n", __func__, size, align);
-
-    if (__offload_myoLoadLibrary()) {
-        return myo_wrapper.SharedAlignedMalloc(size, align);
-    }
-    else {
-        if (align < sizeof(void*)) {
-            align = sizeof(void*);
-        }
-        return _mm_malloc(size, align);
-    }
-}
-
-extern "C" void _Offload_shared_aligned_free(void *ptr)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
-
-    if (__offload_myoLoadLibrary()) {
-        myo_wrapper.SharedAlignedFree(ptr);
-    }
-    else {
-        _mm_free(ptr);
-    }
-}
-
-extern "C" void __intel_cilk_for_32_offload(
-    int size,
-    void (*copy_constructor)(void*, void*),
-    int target_number,
-    void *raddr,
-    void *closure_object,
-    unsigned int iters,
-    unsigned int grain_size)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-    target_number = __offload_myoIsAvailable(target_number);
-    if (target_number >= 0) {
-        struct S {
-            void *M1;
-            unsigned int M2;
-            unsigned int M3;
-            char closure[];
-        } *args;
-
-        args = (struct S*) _Offload_shared_malloc(sizeof(struct S) + size);
-        args->M1 = raddr;
-        args->M2 = iters;
-        args->M3 = grain_size;
-
-        if (copy_constructor == 0) {
-            memcpy(args->closure, closure_object, size);
-        }
-        else {
-            copy_constructor(args->closure, closure_object);
-        }
-
-        myo_wrapper.Release();
-        myo_wrapper.GetResult(
-            myo_wrapper.RemoteCall("__intel_cilk_for_32_offload",
-                                   args, target_number)
-        );
-        myo_wrapper.Acquire();
-
-        _Offload_shared_free(args);
-
-        ORSL::release(target_number);
-    }
-    else {
-        __cilkrts_cilk_for_32(raddr,
-                              closure_object,
-                              iters,
-                              grain_size);
-    }
-}
-
-extern "C" void __intel_cilk_for_64_offload(
-    int size,
-    void (*copy_constructor)(void*, void*),
-    int target_number,
-    void *raddr,
-    void *closure_object,
-    uint64_t iters,
-    uint64_t grain_size)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-    target_number = __offload_myoIsAvailable(target_number);
-    if (target_number >= 0) {
-        struct S {
-            void *M1;
-            uint64_t M2;
-            uint64_t M3;
-            char closure[];
-        } *args;
-
-        args = (struct S*) _Offload_shared_malloc(sizeof(struct S) + size);
-        args->M1 = raddr;
-        args->M2 = iters;
-        args->M3 = grain_size;
-
-        if (copy_constructor == 0) {
-            memcpy(args->closure, closure_object, size);
-        }
-        else {
-            copy_constructor(args->closure, closure_object);
-        }
-
-        myo_wrapper.Release();
-        myo_wrapper.GetResult(
-            myo_wrapper.RemoteCall("__intel_cilk_for_64_offload", args,
-                                   target_number)
-        );
-        myo_wrapper.Acquire();
-
-        _Offload_shared_free(args);
-
-        ORSL::release(target_number);
-    }
-    else {
-        __cilkrts_cilk_for_64(raddr,
-                              closure_object,
-                              iters,
-                              grain_size);
-    }
-}

offload/src/offload_myo_host.h

@@ -1,80 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef OFFLOAD_MYO_HOST_H_INCLUDED
-#define OFFLOAD_MYO_HOST_H_INCLUDED
-
-#include <myotypes.h>
-#include <myoimpl.h>
-#include <myo.h>
-#include "offload.h"
-
-typedef MyoiSharedVarEntry      SharedTableEntry;
-//typedef MyoiHostSharedFptrEntry FptrTableEntry;
-typedef struct {
-    //! Function Name
-    const char *funcName;
-    //! Function Address
-    void *funcAddr;
-    //! Local Thunk Address
-    void *localThunkAddr;
-#ifdef TARGET_WINNT
-    // Dummy to pad up to 32 bytes
-    void *dummy;
-#endif // TARGET_WINNT
-} FptrTableEntry;
-
-struct InitTableEntry {
-#ifdef TARGET_WINNT
-    // Dummy to pad up to 16 bytes
-    // Function Name
-    const char *funcName;
-#endif // TARGET_WINNT
-    void (*func)(void);
-};
-
-#ifdef TARGET_WINNT
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_START          ".MyoSharedTable$a"
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_END            ".MyoSharedTable$z"
-
-#define OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_START     ".MyoSharedInitTable$a"
-#define OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_END       ".MyoSharedInitTable$z"
-
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_START            ".MyoFptrTable$a"
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_END              ".MyoFptrTable$z"
-#else  // TARGET_WINNT
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_START          ".MyoSharedTable."
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_END            ".MyoSharedTable."
-
-#define OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_START     ".MyoSharedInitTable."
-#define OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_END       ".MyoSharedInitTable."
-
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_START            ".MyoFptrTable."
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_END              ".MyoFptrTable."
-#endif // TARGET_WINNT
-
-#pragma section(OFFLOAD_MYO_SHARED_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_MYO_SHARED_TABLE_SECTION_END, read, write)
-
-#pragma section(OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_MYO_SHARED_INIT_TABLE_SECTION_END, read, write)
-
-#pragma section(OFFLOAD_MYO_FPTR_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_MYO_FPTR_TABLE_SECTION_END, read, write)
-
-extern "C" void __offload_myoRegisterTables(
-    InitTableEntry *init_table,
-    SharedTableEntry *shared_table,
-    FptrTableEntry *fptr_table
-);
-
-extern void __offload_myoFini(void);
-
-#endif // OFFLOAD_MYO_HOST_H_INCLUDED

offload/src/offload_myo_target.cpp

@@ -1,184 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_myo_target.h"
-#include "offload_target.h"
-
-extern "C" void __cilkrts_cilk_for_32(void*, void*, uint32_t, int32_t);
-extern "C" void __cilkrts_cilk_for_64(void*, void*, uint64_t, int32_t);
-
-#pragma weak __cilkrts_cilk_for_32
-#pragma weak __cilkrts_cilk_for_64
-
-static void CheckResult(const char *func, MyoError error) {
-    if (error != MYO_SUCCESS) {
-       LIBOFFLOAD_ERROR(c_myotarget_checkresult, func, error);
-        exit(1);
-    }
-}
-
-static void __offload_myo_shared_table_register(SharedTableEntry *entry)
-{
-    int entries = 0;
-    SharedTableEntry *t_start;
-
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    t_start = entry;
-    while (t_start->varName != 0) {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_mic_myo_shared,
-                              "myo shared entry name = \"%s\" addr = %p\n",
-                              t_start->varName, t_start->sharedAddr);
-        t_start++;
-        entries++;
-    }
-
-    if (entries > 0) {
-        OFFLOAD_DEBUG_TRACE(3, "myoiMicVarTableRegister(%p, %d)\n", entry,
-                            entries);
-        CheckResult("myoiMicVarTableRegister",
-                    myoiMicVarTableRegister(entry, entries));
-    }
-}
-
-static void __offload_myo_fptr_table_register(
-    FptrTableEntry *entry
-)
-{
-    int entries = 0;
-    FptrTableEntry *t_start;
-
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
-
-    t_start = entry;
-    while (t_start->funcName != 0) {
-        OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_mic_myo_fptr,
-                              "myo fptr entry name = \"%s\" addr = %p\n",
-                              t_start->funcName, t_start->funcAddr);
-        t_start++;
-        entries++;
-    }
-
-    if (entries > 0) {
-        OFFLOAD_DEBUG_TRACE(3, "myoiTargetFptrTableRegister(%p, %d, 0)\n",
-                            entry, entries);
-        CheckResult("myoiTargetFptrTableRegister",
-                    myoiTargetFptrTableRegister(entry, entries, 0));
-    }
-}
-
-extern "C" void __offload_myoAcquire(void)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-    CheckResult("myoAcquire", myoAcquire());
-}
-
-extern "C" void __offload_myoRelease(void)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-    CheckResult("myoRelease", myoRelease());
-}
-
-extern "C" void __intel_cilk_for_32_offload_wrapper(void *args_)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-    struct S {
-        void *M1;
-        unsigned int M2;
-        unsigned int M3;
-        char closure[];
-    } *args = (struct S*) args_;
-
-    __cilkrts_cilk_for_32(args->M1, args->closure, args->M2, args->M3);
-}
-
-extern "C" void __intel_cilk_for_64_offload_wrapper(void *args_)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-    struct S {
-        void *M1;
-        uint64_t M2;
-        uint64_t M3;
-        char closure[];
-    } *args = (struct S*) args_;
-
-    __cilkrts_cilk_for_64(args->M1, args->closure, args->M2, args->M3);
-}
-
-static void __offload_myo_once_init(void)
-{
-    CheckResult("myoiRemoteFuncRegister",
-                myoiRemoteFuncRegister(
-                    (MyoiRemoteFuncType) __intel_cilk_for_32_offload_wrapper,
-                    "__intel_cilk_for_32_offload"));
-    CheckResult("myoiRemoteFuncRegister",
-                myoiRemoteFuncRegister(
-                    (MyoiRemoteFuncType) __intel_cilk_for_64_offload_wrapper,
-                    "__intel_cilk_for_64_offload"));
-}
-
-extern "C" void __offload_myoRegisterTables(
-    SharedTableEntry *shared_table,
-    FptrTableEntry *fptr_table
-)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
-
-    // one time registration of Intel(R) Cilk(TM) language entries
-    static pthread_once_t once_control = PTHREAD_ONCE_INIT;
-    pthread_once(&once_control, __offload_myo_once_init);
-
-    // register module's tables
-    if (shared_table->varName == 0 && fptr_table->funcName == 0) {
-        return;
-    }
-
-    __offload_myo_shared_table_register(shared_table);
-    __offload_myo_fptr_table_register(fptr_table);
-}
-
-extern "C" void* _Offload_shared_malloc(size_t size)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%lld)\n", __func__, size);
-    return myoSharedMalloc(size);
-}
-
-extern "C" void _Offload_shared_free(void *ptr)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
-    myoSharedFree(ptr);
-}
-
-extern "C" void* _Offload_shared_aligned_malloc(size_t size, size_t align)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%lld, %lld)\n", __func__, size, align);
-    return myoSharedAlignedMalloc(size, align);
-}
-
-extern "C" void _Offload_shared_aligned_free(void *ptr)
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
-    myoSharedAlignedFree(ptr);
-}
-
-// temporary workaround for blocking behavior of myoiLibInit/Fini calls
-extern "C" void __offload_myoLibInit()
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s()\n", __func__);
-    CheckResult("myoiLibInit", myoiLibInit(0, 0));
-}
-
-extern "C" void __offload_myoLibFini()
-{
-    OFFLOAD_DEBUG_TRACE(3, "%s()\n", __func__);
-    myoiLibFini();
-}

offload/src/offload_myo_target.h

@@ -1,54 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef OFFLOAD_MYO_TARGET_H_INCLUDED
-#define OFFLOAD_MYO_TARGET_H_INCLUDED
-
-#include <myotypes.h>
-#include <myoimpl.h>
-#include <myo.h>
-#include "offload.h"
-
-typedef MyoiSharedVarEntry          SharedTableEntry;
-typedef MyoiTargetSharedFptrEntry   FptrTableEntry;
-
-#ifdef TARGET_WINNT
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_START          ".MyoSharedTable$a"
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_END            ".MyoSharedTable$z"
-
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_START            ".MyoFptrTable$a"
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_END              ".MyoFptrTable$z"
-#else  // TARGET_WINNT
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_START          ".MyoSharedTable."
-#define OFFLOAD_MYO_SHARED_TABLE_SECTION_END            ".MyoSharedTable."
-
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_START            ".MyoFptrTable."
-#define OFFLOAD_MYO_FPTR_TABLE_SECTION_END              ".MyoFptrTable."
-#endif // TARGET_WINNT
-
-#pragma section(OFFLOAD_MYO_SHARED_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_MYO_SHARED_TABLE_SECTION_END, read, write)
-
-#pragma section(OFFLOAD_MYO_FPTR_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_MYO_FPTR_TABLE_SECTION_END, read, write)
-
-extern "C" void __offload_myoRegisterTables(
-    SharedTableEntry *shared_table,
-    FptrTableEntry *fptr_table
-);
-
-extern "C" void __offload_myoAcquire(void);
-extern "C" void __offload_myoRelease(void);
-
-// temporary workaround for blocking behavior for myoiLibInit/Fini calls
-extern "C" void __offload_myoLibInit();
-extern "C" void __offload_myoLibFini();
-
-#endif // OFFLOAD_MYO_TARGET_H_INCLUDED

offload/src/offload_omp_host.cpp

@@ -1,851 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include <omp.h>
-#include "offload.h"
-#include "compiler_if_host.h"
-
-// OpenMP API
-
-void omp_set_default_device(int num)
-{
-    if (num >= 0) {
-        __omp_device_num = num;
-    }
-}
-
-int omp_get_default_device(void)
-{
-    return __omp_device_num;
-}
-
-int omp_get_num_devices()
-{
-    __offload_init_library();
-    return mic_engines_total;
-}
-
-// OpenMP API wrappers
-
-static void omp_set_int_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int setting,
-    const char* f_name
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          f_name, 0);
-    if (ofld) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(int);
-        vars[0].count = 1;
-        vars[0].ptr = &setting;
-
-        OFFLOAD_OFFLOAD(ofld, f_name, 0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-static int omp_get_int_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    const char * f_name
-)
-{
-    int setting = 0;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          f_name, 0);
-    if (ofld) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_out;
-        vars[0].size = sizeof(int);
-        vars[0].count = 1;
-        vars[0].ptr = &setting;
-
-        OFFLOAD_OFFLOAD(ofld, f_name, 0, 1, vars, NULL, 0, 0, 0);
-    }
-    return setting;
-}
-
-void omp_set_num_threads_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-)
-{
-    omp_set_int_target(target_type, target_number, num_threads,
-                       "omp_set_num_threads_target");
-}
-
-int omp_get_max_threads_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "omp_get_max_threads_target");
-}
-
-int omp_get_num_procs_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "omp_get_num_procs_target");
-}
-
-void omp_set_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int num_threads
-)
-{
-    omp_set_int_target(target_type, target_number, num_threads,
-                       "omp_set_dynamic_target");
-}
-
-int omp_get_dynamic_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "omp_get_dynamic_target");
-}
-
-void omp_set_nested_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int nested
-)
-{
-    omp_set_int_target(target_type, target_number, nested,
-                       "omp_set_nested_target");
-}
-
-int omp_get_nested_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "omp_get_nested_target");
-}
-
-void omp_set_schedule_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_sched_t kind,
-    int modifier
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(omp_sched_t);
-        vars[0].count = 1;
-        vars[0].ptr = &kind;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_in;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = &modifier;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_set_schedule_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_get_schedule_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_sched_t *kind,
-    int *modifier
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_out;
-        vars[0].size = sizeof(omp_sched_t);
-        vars[0].count = 1;
-        vars[0].ptr = kind;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_out;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = modifier;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_get_schedule_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-}
-
-// lock API functions
-
-void omp_init_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_out;
-        vars[0].size = sizeof(omp_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_init_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_destroy_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(omp_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_destroy_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_set_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_set_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_unset_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_unset_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-int omp_test_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_lock_target_t *lock
-)
-{
-    int result = 0;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_out;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_test_lock_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-// nested lock API functions
-
-void omp_init_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_out;
-        vars[0].size = sizeof(omp_nest_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_init_nest_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_destroy_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(omp_nest_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_destroy_nest_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_set_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_nest_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_set_nest_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void omp_unset_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_nest_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_unset_nest_lock_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-int omp_test_nest_lock_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    omp_nest_lock_target_t *lock
-)
-{
-    int result = 0;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(omp_nest_lock_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = lock;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_out;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "omp_test_nest_lock_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-// kmp API functions
-
-void kmp_set_stacksize_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int size
-)
-{
-    omp_set_int_target(target_type, target_number, size,
-                       "kmp_set_stacksize_target");
-}
-
-int kmp_get_stacksize_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "kmp_get_stacksize_target");
-}
-
-void kmp_set_stacksize_s_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    size_t size
-)
-{
-    omp_set_int_target(target_type, target_number, size,
-                       "kmp_set_stacksize_s_target");
-}
-
-size_t kmp_get_stacksize_s_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "kmp_get_stacksize_s_target");
-}
-
-void kmp_set_blocktime_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int time
-)
-{
-    omp_set_int_target(target_type, target_number, time,
-                       "kmp_set_blocktime_target");
-}
-
-int kmp_get_blocktime_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "kmp_get_blocktime_target");
-}
-
-void kmp_set_library_serial_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_library_serial_target",
-                        0, 0, 0, 0, 0, 0, 0);
-    }
-}
-
-void kmp_set_library_turnaround_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_library_turnaround_target",
-                        0, 0, 0, 0, 0, 0, 0);
-    }
-}
-
-void kmp_set_library_throughput_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_library_throughput_target",
-                        0, 0, 0, 0, 0, 0, 0);
-    }
-}
-
-void kmp_set_library_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int mode
-)
-{
-    omp_set_int_target(target_type, target_number, mode,
-                       "kmp_set_library_target");
-}
-
-int kmp_get_library_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "kmp_get_library_target");
-}
-
-void kmp_set_defaults_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    char const *defaults
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_string_ptr;
-        vars[0].type.dst = c_string_ptr;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].alloc_if = 1;
-        vars[0].free_if = 1;
-        vars[0].ptr = &defaults;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_defaults_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-// affinity API functions
-
-void kmp_create_affinity_mask_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_out;
-        vars[0].size = sizeof(kmp_affinity_mask_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = mask;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_create_affinity_mask_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-void kmp_destroy_affinity_mask_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[1] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(kmp_affinity_mask_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = mask;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_destroy_affinity_mask_target",
-                        0, 1, vars, NULL, 0, 0, 0);
-    }
-}
-
-int kmp_set_affinity_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    int result = 1;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(kmp_affinity_mask_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = mask;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_out;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_affinity_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-int kmp_get_affinity_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    int result = 1;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[2] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_inout;
-        vars[0].size = sizeof(kmp_affinity_mask_target_t);
-        vars[0].count = 1;
-        vars[0].ptr = mask;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_out;
-        vars[1].size = sizeof(int);
-        vars[1].count = 1;
-        vars[1].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_get_affinity_target",
-                        0, 2, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-int kmp_get_affinity_max_proc_target(
-    TARGET_TYPE target_type,
-    int target_number
-)
-{
-    return omp_get_int_target(target_type, target_number,
-                              "kmp_get_affinity_max_proc_target");
-}
-
-int kmp_set_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    int result = 1;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[3] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(int);
-        vars[0].count = 1;
-        vars[0].ptr = &proc;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_inout;
-        vars[1].size = sizeof(kmp_affinity_mask_target_t);
-        vars[1].count = 1;
-        vars[1].ptr = mask;
-
-        vars[2].type.src = c_data;
-        vars[2].type.dst = c_data;
-        vars[2].direction.bits = c_parameter_out;
-        vars[2].size = sizeof(int);
-        vars[2].count = 1;
-        vars[2].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_set_affinity_mask_proc_target",
-                        0, 3, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-int kmp_unset_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    int result = 1;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[3] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(int);
-        vars[0].count = 1;
-        vars[0].ptr = &proc;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_inout;
-        vars[1].size = sizeof(kmp_affinity_mask_target_t);
-        vars[1].count = 1;
-        vars[1].ptr = mask;
-
-        vars[2].type.src = c_data;
-        vars[2].type.dst = c_data;
-        vars[2].direction.bits = c_parameter_out;
-        vars[2].size = sizeof(int);
-        vars[2].count = 1;
-        vars[2].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_unset_affinity_mask_proc_target",
-                        0, 3, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}
-
-int kmp_get_affinity_mask_proc_target(
-    TARGET_TYPE target_type,
-    int target_number,
-    int proc,
-    kmp_affinity_mask_target_t *mask
-)
-{
-    int result = 1;
-
-    OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL,
-                                          __func__, 0);
-    if (ofld != 0) {
-        VarDesc vars[3] = {0};
-
-        vars[0].type.src = c_data;
-        vars[0].type.dst = c_data;
-        vars[0].direction.bits = c_parameter_in;
-        vars[0].size = sizeof(int);
-        vars[0].count = 1;
-        vars[0].ptr = &proc;
-
-        vars[1].type.src = c_data;
-        vars[1].type.dst = c_data;
-        vars[1].direction.bits = c_parameter_in;
-        vars[1].size = sizeof(kmp_affinity_mask_target_t);
-        vars[1].count = 1;
-        vars[1].ptr = mask;
-
-        vars[2].type.src = c_data;
-        vars[2].type.dst = c_data;
-        vars[2].direction.bits = c_parameter_out;
-        vars[2].size = sizeof(int);
-        vars[2].count = 1;
-        vars[2].ptr = &result;
-
-        OFFLOAD_OFFLOAD(ofld, "kmp_get_affinity_mask_proc_target",
-                        0, 3, vars, NULL, 0, 0, 0);
-    }
-    return result;
-}

offload/src/offload_orsl.cpp

@@ -1,84 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_orsl.h"
-#include <stdlib.h>
-#include "offload_host.h"
-#include "orsl-lite/include/orsl-lite.h"
-
-namespace ORSL {
-
-static bool            is_enabled = false;
-static const ORSLTag   my_tag = "Offload";
-
-void init()
-{
-    const char *env_var = getenv("OFFLOAD_ENABLE_ORSL");
-    if (env_var != 0 && *env_var != '\0') {
-        int64_t new_val;
-        if (__offload_parse_int_string(env_var, new_val)) {
-            is_enabled = new_val;
-        }
-        else {
-            LIBOFFLOAD_ERROR(c_invalid_env_var_int_value,
-                             "OFFLOAD_ENABLE_ORSL");
-        }
-    }
-
-    if (is_enabled) {
-        OFFLOAD_DEBUG_TRACE(2, "ORSL is enabled\n");
-    }
-    else {
-        OFFLOAD_DEBUG_TRACE(2, "ORSL is disabled\n");
-    }
-}
-
-bool reserve(int device)
-{
-    if (is_enabled) {
-        int pnum = mic_engines[device].get_physical_index();
-        ORSLBusySet bset;
-
-        bset.type = BUSY_SET_FULL;
-        if (ORSLReserve(1, &pnum, &bset, my_tag) != 0) {
-            return false;
-        }
-    }
-    return true;
-}
-
-bool try_reserve(int device)
-{
-    if (is_enabled) {
-        int pnum = mic_engines[device].get_physical_index();
-        ORSLBusySet bset;
-
-        bset.type = BUSY_SET_FULL;
-        if (ORSLTryReserve(1, &pnum, &bset, my_tag) != 0) {
-            return false;
-        }
-    }
-    return true;
-}
-
-void release(int device)
-{
-    if (is_enabled) {
-        int pnum = mic_engines[device].get_physical_index();
-        ORSLBusySet bset;
-
-        bset.type = BUSY_SET_FULL;
-        if (ORSLRelease(1, &pnum, &bset, my_tag) != 0) {
-            // should never get here
-        }
-    }
-}
-
-} // namespace ORSL

offload/src/offload_orsl.h

@@ -1,25 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#ifndef OFFLOAD_ORSL_H_INCLUDED
-#define OFFLOAD_ORSL_H_INCLUDED
-
-// ORSL interface
-namespace ORSL {
-
-extern void init();
-
-extern bool reserve(int device);
-extern bool try_reserve(int device);
-extern void release(int device);
-
-} // namespace ORSL
-
-#endif // OFFLOAD_ORSL_H_INCLUDED

offload/src/offload_table.cpp

@@ -1,375 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "offload_table.h"
-#include "offload_common.h"
-
-#if !HOST_LIBRARY
-// Predefined offload entries
-extern void omp_set_num_threads_lrb(void*);
-extern void omp_get_max_threads_lrb(void*);
-extern void omp_get_num_procs_lrb(void*);
-extern void omp_set_dynamic_lrb(void*);
-extern void omp_get_dynamic_lrb(void*);
-extern void omp_set_nested_lrb(void*);
-extern void omp_get_nested_lrb(void*);
-extern void omp_set_schedule_lrb(void*);
-extern void omp_get_schedule_lrb(void*);
-
-extern void omp_init_lock_lrb(void*);
-extern void omp_destroy_lock_lrb(void*);
-extern void omp_set_lock_lrb(void*);
-extern void omp_unset_lock_lrb(void*);
-extern void omp_test_lock_lrb(void*);
-
-extern void omp_init_nest_lock_lrb(void*);
-extern void omp_destroy_nest_lock_lrb(void*);
-extern void omp_set_nest_lock_lrb(void*);
-extern void omp_unset_nest_lock_lrb(void*);
-extern void omp_test_nest_lock_lrb(void*);
-
-extern void kmp_set_stacksize_lrb(void*);
-extern void kmp_get_stacksize_lrb(void*);
-extern void kmp_set_stacksize_s_lrb(void*);
-extern void kmp_get_stacksize_s_lrb(void*);
-extern void kmp_set_blocktime_lrb(void*);
-extern void kmp_get_blocktime_lrb(void*);
-extern void kmp_set_library_serial_lrb(void*);
-extern void kmp_set_library_turnaround_lrb(void*);
-extern void kmp_set_library_throughput_lrb(void*);
-extern void kmp_set_library_lrb(void*);
-extern void kmp_get_library_lrb(void*);
-extern void kmp_set_defaults_lrb(void*);
-
-extern void kmp_create_affinity_mask_lrb(void*);
-extern void kmp_destroy_affinity_mask_lrb(void*);
-extern void kmp_set_affinity_lrb(void*);
-extern void kmp_get_affinity_lrb(void*);
-extern void kmp_get_affinity_max_proc_lrb(void*);
-extern void kmp_set_affinity_mask_proc_lrb(void*);
-extern void kmp_unset_affinity_mask_proc_lrb(void*);
-extern void kmp_get_affinity_mask_proc_lrb(void*);
-
-// Predefined entries on the target side
-static FuncTable::Entry predefined_entries[] = {
-    "omp_set_num_threads_target",
-    (void*) &omp_set_num_threads_lrb,
-    "omp_get_max_threads_target",
-    (void*) &omp_get_max_threads_lrb,
-    "omp_get_num_procs_target",
-    (void*) &omp_get_num_procs_lrb,
-    "omp_set_dynamic_target",
-    (void*) &omp_set_dynamic_lrb,
-    "omp_get_dynamic_target",
-    (void*) &omp_get_dynamic_lrb,
-    "omp_set_nested_target",
-    (void*) &omp_set_nested_lrb,
-    "omp_get_nested_target",
-    (void*) &omp_get_nested_lrb,
-    "omp_set_schedule_target",
-    (void*) &omp_set_schedule_lrb,
-    "omp_get_schedule_target",
-    (void*) &omp_get_schedule_lrb,
-
-    "omp_init_lock_target",
-    (void*) &omp_init_lock_lrb,
-    "omp_destroy_lock_target",
-    (void*) &omp_destroy_lock_lrb,
-    "omp_set_lock_target",
-    (void*) &omp_set_lock_lrb,
-    "omp_unset_lock_target",
-    (void*) &omp_unset_lock_lrb,
-    "omp_test_lock_target",
-    (void*) &omp_test_lock_lrb,
-
-    "omp_init_nest_lock_target",
-    (void*) &omp_init_nest_lock_lrb,
-    "omp_destroy_nest_lock_target",
-    (void*) &omp_destroy_nest_lock_lrb,
-    "omp_set_nest_lock_target",
-    (void*) &omp_set_nest_lock_lrb,
-    "omp_unset_nest_lock_target",
-    (void*) &omp_unset_nest_lock_lrb,
-    "omp_test_nest_lock_target",
-    (void*) &omp_test_nest_lock_lrb,
-
-    "kmp_set_stacksize_target",
-    (void*) &kmp_set_stacksize_lrb,
-    "kmp_get_stacksize_target",
-    (void*) &kmp_get_stacksize_lrb,
-    "kmp_set_stacksize_s_target",
-    (void*) &kmp_set_stacksize_s_lrb,
-    "kmp_get_stacksize_s_target",
-    (void*) &kmp_get_stacksize_s_lrb,
-    "kmp_set_blocktime_target",
-    (void*) &kmp_set_blocktime_lrb,
-    "kmp_get_blocktime_target",
-    (void*) &kmp_get_blocktime_lrb,
-    "kmp_set_library_serial_target",
-    (void*) &kmp_set_library_serial_lrb,
-    "kmp_set_library_turnaround_target",
-    (void*) &kmp_set_library_turnaround_lrb,
-    "kmp_set_library_throughput_target",
-    (void*) &kmp_set_library_throughput_lrb,
-    "kmp_set_library_target",
-    (void*) &kmp_set_library_lrb,
-    "kmp_get_library_target",
-    (void*) &kmp_get_library_lrb,
-    "kmp_set_defaults_target",
-    (void*) &kmp_set_defaults_lrb,
-
-    "kmp_create_affinity_mask_target",
-    (void*) &kmp_create_affinity_mask_lrb,
-    "kmp_destroy_affinity_mask_target",
-    (void*) &kmp_destroy_affinity_mask_lrb,
-    "kmp_set_affinity_target",
-    (void*) &kmp_set_affinity_lrb,
-    "kmp_get_affinity_target",
-    (void*) &kmp_get_affinity_lrb,
-    "kmp_get_affinity_max_proc_target",
-    (void*) &kmp_get_affinity_max_proc_lrb,
-    "kmp_set_affinity_mask_proc_target",
-    (void*) &kmp_set_affinity_mask_proc_lrb,
-    "kmp_unset_affinity_mask_proc_target",
-    (void*) &kmp_unset_affinity_mask_proc_lrb,
-    "kmp_get_affinity_mask_proc_target",
-    (void*) &kmp_get_affinity_mask_proc_lrb,
-
-    (const char*) -1,
-    (void*) -1
-};
-
-static FuncList::Node predefined_table = {
-    { predefined_entries, -1 },
-    0, 0
-};
-
-// Entry table
-FuncList __offload_entries(&predefined_table);
-#else
-FuncList __offload_entries;
-#endif // !HOST_LIBRARY
-
-// Function table. No predefined entries.
-FuncList __offload_funcs;
-
-// Var table
-VarList  __offload_vars;
-
-// Given the function name returns the associtated function pointer
-const void* FuncList::find_addr(const char *name)
-{
-    const void* func = 0;
-
-    m_lock.lock();
-
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->name != 0 && strcmp(e->name, name) == 0) {
-                func = e->func;
-                break;
-            }
-        }
-    }
-
-    m_lock.unlock();
-
-    return func;
-}
-
-// Given the function pointer returns the associtated function name
-const char* FuncList::find_name(const void *func)
-{
-    const char* name = 0;
-
-    m_lock.lock();
-
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->func == func) {
-                name = e->name;
-                break;
-            }
-        }
-    }
-
-    m_lock.unlock();
-
-    return name;
-}
-
-// Returns max name length from all tables
-int64_t FuncList::max_name_length(void)
-{
-    if (m_max_name_len < 0) {
-        m_lock.lock();
-
-        m_max_name_len = 0;
-        for (Node *n = m_head; n != 0; n = n->next) {
-            if (n->table.max_name_len < 0) {
-                n->table.max_name_len = 0;
-
-                // calculate max name length in a single table
-                for (const Table::Entry *e = n->table.entries;
-                     e->name != (const char*) -1; e++) {
-                    if (e->name != 0) {
-                        size_t len = strlen(e->name) + 1;
-                        if (n->table.max_name_len < len) {
-                            n->table.max_name_len = len;
-                        }
-                    }
-                }
-            }
-
-            // select max from all tables
-            if (m_max_name_len < n->table.max_name_len) {
-                m_max_name_len = n->table.max_name_len;
-            }
-        }
-
-        m_lock.unlock();
-    }
-    return m_max_name_len;
-}
-
-// Debugging dump
-void FuncList::dump(void)
-{
-    OFFLOAD_DEBUG_TRACE(2, "Function table:\n");
-
-    m_lock.lock();
-
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->name != 0) {
-                OFFLOAD_DEBUG_TRACE(2, "%p %s\n", e->func, e->name);
-            }
-        }
-    }
-
-    m_lock.unlock();
-}
-
-// Debugging dump
-void VarList::dump(void)
-{
-    OFFLOAD_DEBUG_TRACE(2, "Var table:\n");
-
-    m_lock.lock();
-
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->name != 0) {
-#if HOST_LIBRARY
-                OFFLOAD_DEBUG_TRACE(2, "%s %p %ld\n", e->name, e->addr,
-                                    e->size);
-#else  // HOST_LIBRARY
-                OFFLOAD_DEBUG_TRACE(2, "%s %p\n", e->name, e->addr);
-#endif // HOST_LIBRARY
-            }
-        }
-    }
-
-    m_lock.unlock();
-}
-
-//
-int64_t VarList::table_size(int64_t &nelems)
-{
-    int64_t length = 0;
-
-    nelems = 0;
-
-    // calculate string table size and number of elements
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->name != 0) {
-                length += strlen(e->name) + 1;
-                nelems++;
-            }
-        }
-    }
-
-    return nelems * sizeof(BufEntry) + length;
-}
-
-// copy table to the gven buffer
-void VarList::table_copy(void *buf, int64_t nelems)
-{
-    BufEntry* elems = static_cast<BufEntry*>(buf);
-    char*     names = reinterpret_cast<char*>(elems + nelems);
-
-    // copy entries to buffer
-    for (Node *n = m_head; n != 0; n = n->next) {
-        for (const Table::Entry *e = n->table.entries;
-             e->name != (const char*) -1; e++) {
-            if (e->name != 0) {
-                // name field contains offset to the name from the beginning
-                // of the buffer
-                elems->name = names - static_cast<char*>(buf);
-                elems->addr = reinterpret_cast<intptr_t>(e->addr);
-
-                // copy name to string table
-                const char *name = e->name;
-                while ((*names++ = *name++) != '\0');
-
-                elems++;
-            }
-        }
-    }
-}
-
-// patch name offsets in a buffer
-void VarList::table_patch_names(void *buf, int64_t nelems)
-{
-    BufEntry* elems = static_cast<BufEntry*>(buf);
-    for (int i = 0; i < nelems; i++) {
-        elems[i].name += reinterpret_cast<intptr_t>(buf);
-    }
-}
-
-// Adds given list element to the global lookup table list
-extern "C" void __offload_register_tables(
-    FuncList::Node *entry_table,
-    FuncList::Node *func_table,
-    VarList::Node *var_table
-)
-{
-    OFFLOAD_DEBUG_TRACE(2, "Registering offload function entry table %p\n",
-                           entry_table);
-    __offload_entries.add_table(entry_table);
-
-    OFFLOAD_DEBUG_TRACE(2, "Registering function table %p\n", func_table);
-    __offload_funcs.add_table(func_table);
-
-    OFFLOAD_DEBUG_TRACE(2, "Registering var table %p\n", var_table);
-    __offload_vars.add_table(var_table);
-}
-
-// Removes given list element from the global lookup table list
-extern "C" void __offload_unregister_tables(
-    FuncList::Node *entry_table,
-    FuncList::Node *func_table,
-    VarList::Node *var_table
-)
-{
-    __offload_entries.remove_table(entry_table);
-
-    OFFLOAD_DEBUG_TRACE(2, "Unregistering function table %p\n", func_table);
-    __offload_funcs.remove_table(func_table);
-
-    OFFLOAD_DEBUG_TRACE(2, "Unregistering var table %p\n", var_table);
-    __offload_vars.remove_table(var_table);
-}

offload/src/offload_table.h

@@ -1,301 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*! \file
-    \brief Function and Variable tables used by the runtime library
-*/
-
-#ifndef OFFLOAD_TABLE_H_INCLUDED
-#define OFFLOAD_TABLE_H_INCLUDED
-
-#include <iterator>
-#include "offload_util.h"
-
-// Template representing double linked list of tables
-template <typename T> class TableList {
-public:
-    // table type
-    typedef T Table;
-
-    // List node
-    struct Node {
-        Table   table;
-        Node*   prev;
-        Node*   next;
-    };
-
-public:
-    explicit TableList(Node *node = 0) : m_head(node) {}
-
-    void add_table(Node *node) {
-        m_lock.lock();
-
-        if (m_head != 0) {
-            node->next = m_head;
-            m_head->prev = node;
-        }
-        m_head = node;
-
-        m_lock.unlock();
-    }
-
-    void remove_table(Node *node) {
-        m_lock.lock();
-
-        if (node->next != 0) {
-            node->next->prev = node->prev;
-        }
-        if (node->prev != 0) {
-            node->prev->next = node->next;
-        }
-        if (m_head == node) {
-            m_head = node->next;
-        }
-
-        m_lock.unlock();
-    }
-
-protected:
-    Node*           m_head;
-    mutex_t         m_lock;
-};
-
-// Function lookup table.
-struct FuncTable {
-    //! Function table entry
-    /*! This table contains functions created from offload regions.   */
-    /*! Each entry consists of a pointer to the function's "key"
-        and the function address.                                     */
-    /*! Each shared library or executable may contain one such table. */
-    /*! The end of the table is marked with an entry whose name field
-        has value -1.                                                 */
-    struct Entry {
-        const char* name; //!< Name of the function
-        void*       func; //!< Address of the function
-    };
-
-    // entries
-    const Entry *entries;
-
-    // max name length
-    int64_t max_name_len;
-};
-
-// Function table
-class FuncList : public TableList<FuncTable> {
-public:
-    explicit FuncList(Node *node = 0) : TableList<Table>(node),
-                                        m_max_name_len(-1)
-    {}
-
-    // add table to the list
-    void add_table(Node *node) {
-        // recalculate max function name length
-        m_max_name_len = -1;
-
-        // add table
-        TableList<Table>::add_table(node);
-    }
-
-    // find function address for the given name
-    const void* find_addr(const char *name);
-
-    // find function name for the given address
-    const char* find_name(const void *addr);
-
-    // max name length from all tables in the list
-    int64_t max_name_length(void);
-
-    // debug dump
-    void dump(void);
-
-private:
-    // max name length within from all tables
-    int64_t m_max_name_len;
-};
-
-// Table entry for static variables
-struct VarTable {
-    //! Variable table entry
-    /*! This table contains statically allocated variables marked with
-        __declspec(target(mic) or #pragma omp declare target.           */
-    /*! Each entry consists of a pointer to the variable's "key",
-        the variable address and its size in bytes.                     */
-    /*! Because memory allocation is done from the host,
-        the MIC table does not need the size of the variable.           */
-    /*! Padding to make the table entry size a power of 2 is necessary
-        to avoid "holes" between table contributions from different object
-        files on Windows when debug information is specified with /Zi.  */
-    struct Entry {
-        const char* name; //!< Name of the variable
-        void*       addr; //!< Address of the variable
-
-#if HOST_LIBRARY
-        uint64_t    size;
-
-#ifdef TARGET_WINNT
-		// padding to make entry size a power of 2
-        uint64_t    padding;
-#endif // TARGET_WINNT
-#endif
-    };
-
-    // Table terminated by an entry with name == -1
-    const Entry *entries;
-};
-
-// List of var tables
-class VarList : public TableList<VarTable> {
-public:
-    VarList() : TableList<Table>()
-    {}
-
-    // debug dump
-    void dump();
-
-public:
-    // var table list iterator
-    class Iterator : public std::iterator<std::input_iterator_tag,
-                                          Table::Entry> {
-    public:
-        Iterator() : m_node(0), m_entry(0) {}
-
-        explicit Iterator(Node *node) {
-            new_node(node);
-        }
-
-        Iterator& operator++() {
-            if (m_entry != 0) {
-                m_entry++;
-                while (m_entry->name == 0) {
-                    m_entry++;
-                }
-                if (m_entry->name == reinterpret_cast<const char*>(-1)) {
-                    new_node(m_node->next);
-                }
-            }
-            return *this;
-        }
-
-        bool operator==(const Iterator &other) const {
-            return m_entry == other.m_entry;
-        }
-
-        bool operator!=(const Iterator &other) const {
-            return m_entry != other.m_entry;
-        }
-
-        const Table::Entry* operator*() const {
-            return m_entry;
-        }
-
-    private:
-        void new_node(Node *node) {
-            m_node = node;
-            m_entry = 0;
-            while (m_node != 0) {
-                m_entry = m_node->table.entries;
-                while (m_entry->name == 0) {
-                    m_entry++;
-                }
-                if (m_entry->name != reinterpret_cast<const char*>(-1)) {
-                    break;
-                }
-                m_node = m_node->next;
-                m_entry = 0;
-            }
-        }
-
-    private:
-        Node                *m_node;
-        const Table::Entry  *m_entry;
-    };
-
-    Iterator begin() const {
-        return Iterator(m_head);
-    }
-
-    Iterator end() const {
-        return Iterator();
-    }
-
-public:
-    // Entry representation in a copy buffer
-    struct BufEntry {
-        intptr_t name;
-        intptr_t addr;
-    };
-
-    // Calculate the number of elements in the table and
-    // returns the size of buffer for the table
-    int64_t table_size(int64_t &nelems);
-
-    // Copy table contents to given buffer. It is supposed to be large
-    // enough to hold all elements as string table.
-    void table_copy(void *buf, int64_t nelems);
-
-    // Patch name offsets in a table after it's been copied to other side
-    static void table_patch_names(void *buf, int64_t nelems);
-};
-
-extern FuncList __offload_entries;
-extern FuncList __offload_funcs;
-extern VarList  __offload_vars;
-
-// Section names where the lookup tables are stored
-#ifdef TARGET_WINNT
-#define OFFLOAD_ENTRY_TABLE_SECTION_START   ".OffloadEntryTable$a"
-#define OFFLOAD_ENTRY_TABLE_SECTION_END     ".OffloadEntryTable$z"
-
-#define OFFLOAD_FUNC_TABLE_SECTION_START    ".OffloadFuncTable$a"
-#define OFFLOAD_FUNC_TABLE_SECTION_END      ".OffloadFuncTable$z"
-
-#define OFFLOAD_VAR_TABLE_SECTION_START     ".OffloadVarTable$a"
-#define OFFLOAD_VAR_TABLE_SECTION_END       ".OffloadVarTable$z"
-
-#define OFFLOAD_CRTINIT_SECTION_START       ".CRT$XCT"
-
-#pragma section(OFFLOAD_CRTINIT_SECTION_START, read)
-
-#else  // TARGET_WINNT
-
-#define OFFLOAD_ENTRY_TABLE_SECTION_START   ".OffloadEntryTable."
-#define OFFLOAD_ENTRY_TABLE_SECTION_END     ".OffloadEntryTable."
-
-#define OFFLOAD_FUNC_TABLE_SECTION_START    ".OffloadFuncTable."
-#define OFFLOAD_FUNC_TABLE_SECTION_END      ".OffloadFuncTable."
-
-#define OFFLOAD_VAR_TABLE_SECTION_START     ".OffloadVarTable."
-#define OFFLOAD_VAR_TABLE_SECTION_END       ".OffloadVarTable."
-#endif // TARGET_WINNT
-
-#pragma section(OFFLOAD_ENTRY_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_ENTRY_TABLE_SECTION_END, read, write)
-
-#pragma section(OFFLOAD_FUNC_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_FUNC_TABLE_SECTION_END, read, write)
-
-#pragma section(OFFLOAD_VAR_TABLE_SECTION_START, read, write)
-#pragma section(OFFLOAD_VAR_TABLE_SECTION_END, read, write)
-
-
-// register/unregister given tables
-extern "C" void __offload_register_tables(
-    FuncList::Node *entry_table,
-    FuncList::Node *func_table,
-    VarList::Node *var_table
-);
-
-extern "C" void __offload_unregister_tables(
-    FuncList::Node *entry_table,
-    FuncList::Node *func_table,
-    VarList::Node *var_table
-);
-#endif  // OFFLOAD_TABLE_H_INCLUDED