Merging r318848:

------------------------------------------------------------------------
r318848 | hahnfeld | 2017-11-22 09:15:20 -0800 (Wed, 22 Nov 2017) | 7 lines

Fix for OMP doacross implementation on Power

Power has a weak consistency model so we need memory barriers to
make writes (both from runtime and from user code) available for
all threads.

Differential Revision: https://reviews.llvm.org/D40175
------------------------------------------------------------------------

git-svn-id: https://llvm.org/svn/llvm-project/openmp/branches/release_50@319057 91177308-0d34-0410-b5e6-96231b3b80d8

CMakeLists.txt

@@ -4,9 +4,20 @@ set(OPENMP_LLVM_TOOLS_DIR "" CACHE PATH "Path to LLVM tools for testing")
 
 add_subdirectory(runtime)
 
+
+# The tests currently don't pass if the generic ELF plugin is built.
+# TODO: Fix the tests and enable libomptarget by default on supported
+#       architectures and platforms.
+set(ENABLE_LIBOMPTARGET OFF)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
-if (NOT (WIN32 OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
+if (WIN32 OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  set(ENABLE_LIBOMPTARGET OFF)
+endif()
+
+option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
+       ${ENABLE_LIBOMPTARGET})
+if (OPENMP_ENABLE_LIBOMPTARGET)
   add_subdirectory(libomptarget)
 endif()

runtime/src/kmp_csupport.cpp

@@ -3095,23 +3095,35 @@ void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims,
     __kmp_wait_yield_4((volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
                        __kmp_eq_4, NULL);
   }
+#if KMP_32_BIT_ARCH
   // Check if we are the first thread. After the CAS the first thread gets 0,
   // others get 1 if initialization is in progress, allocated pointer otherwise.
+  // Treat pointer as volatile integer (value 0 or 1) until memory is allocated.
+  flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
+      (volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
+#else
   flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
-      (kmp_int64 *)&sh_buf->doacross_flags, NULL, (kmp_int64)1);
+      (volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
+#endif
   if (flags == NULL) {
     // we are the first thread, allocate the array of flags
-    kmp_int64 size =
-        trace_count / 8 + 8; // in bytes, use single bit per iteration
-    sh_buf->doacross_flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
-  } else if ((kmp_int64)flags == 1) {
+    size_t size = trace_count / 8 + 8; // in bytes, use single bit per iteration
+    flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
+    KMP_MB();
+    sh_buf->doacross_flags = flags;
+  } else if (flags == (kmp_uint32 *)1) {
+#if KMP_32_BIT_ARCH
     // initialization is still in progress, need to wait
-    while ((volatile kmp_int64)sh_buf->doacross_flags == 1) {
+    while (*(volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
+#else
+    while (*(volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
+#endif
       KMP_YIELD(TRUE);
-    }
+    KMP_MB();
+  } else {
+    KMP_MB();
   }
-  KMP_DEBUG_ASSERT((kmp_int64)sh_buf->doacross_flags >
-                   1); // check value of pointer
+  KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1); // check ptr value
   pr_buf->th_doacross_flags =
       sh_buf->doacross_flags; // save private copy in order to not
   // touch shared buffer on each iteration
@@ -3205,6 +3217,7 @@ void __kmpc_doacross_wait(ident_t *loc, int gtid, long long *vec) {
   while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
     KMP_YIELD(TRUE);
   }
+  KMP_MB();
   KA_TRACE(20,
            ("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
             gtid, (iter_number << 5) + shft));
@@ -3257,6 +3270,7 @@ void __kmpc_doacross_post(ident_t *loc, int gtid, long long *vec) {
   shft = iter_number % 32; // use 32-bit granularity
   iter_number >>= 5; // divided by 32
   flag = 1 << shft;
+  KMP_MB();
   if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
     KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
   KA_TRACE(20, ("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,

runtime/src/kmp_lock.cpp

@@ -3061,11 +3061,12 @@ kmp_indirect_lock_t *__kmp_allocate_indirect_lock(void **user_lock,
     if (idx == __kmp_i_lock_table.size) {
       // Double up the space for block pointers
       int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
-      kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
-      __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
+      kmp_indirect_lock_t **new_table = (kmp_indirect_lock_t **)__kmp_allocate(
           2 * row * sizeof(kmp_indirect_lock_t *));
-      KMP_MEMCPY(__kmp_i_lock_table.table, old_table,
+      KMP_MEMCPY(new_table, __kmp_i_lock_table.table,
                  row * sizeof(kmp_indirect_lock_t *));
+      kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
+      __kmp_i_lock_table.table = new_table;
       __kmp_free(old_table);
       // Allocate new objects in the new blocks
       for (int i = row; i < 2 * row; ++i)

runtime/src/kmp_str.cpp

@@ -168,7 +168,7 @@ void __kmp_str_buf_vprint(kmp_str_buf_t *buffer, char const *format,
 
 #if !KMP_OS_WINDOWS
       va_list _args;
-      __va_copy(_args, args); // Make copy of args.
+      va_copy(_args, args); // Make copy of args.
 #define args _args // Substitute args with its copy, _args.
 #endif // KMP_OS_WINDOWS
       rc = KMP_VSNPRINTF(buffer->str + buffer->used, free, format, args);

runtime/src/kmp_tasking.cpp

@@ -2504,14 +2504,13 @@ static void __kmp_realloc_task_deque(kmp_info_t *thread,
 // Deallocates a task deque for a particular thread. Happens at library
 // deallocation so don't need to reset all thread data fields.
 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
-  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
-
   if (thread_data->td.td_deque != NULL) {
+    __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
     TCW_4(thread_data->td.td_deque_ntasks, 0);
     __kmp_free(thread_data->td.td_deque);
     thread_data->td.td_deque = NULL;
+    __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
   }
-  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
 
 #ifdef BUILD_TIED_TASK_STACK
   // GEH: Figure out what to do here for td_susp_tied_tasks

runtime/src/kmp_threadprivate.cpp

@@ -228,6 +228,13 @@ void __kmp_common_destroy_gtid(int gtid) {
   struct private_common *tn;
   struct shared_common *d_tn;
 
+  if (!TCR_4(__kmp_init_gtid)) {
+    // This is possible when one of multiple roots initiates early library
+    // termination in a sequential region while other teams are active, and its
+    // child threads are about to end.
+    return;
+  }
+
   KC_TRACE(10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid));
   if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) : (!KMP_UBER_GTID(gtid))) {
 

runtime/test/lock/omp_init_lock.c

@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run
+#include "omp_testsuite.h"
+#include <stdio.h>
+
+// This should be slightly less than KMP_I_LOCK_CHUNK, which is 1024
+#define LOCKS_PER_ITER 1000
+#define ITERATIONS (REPETITIONS + 1)
+
+// This tests concurrently using locks on one thread while initializing new
+// ones on another thread.  This exercises the global lock pool.
+int test_omp_init_lock() {
+  int i;
+  omp_lock_t lcks[ITERATIONS * LOCKS_PER_ITER];
+#pragma omp parallel for schedule(static) num_threads(NUM_TASKS)
+  for (i = 0; i < ITERATIONS; i++) {
+    int j;
+    omp_lock_t *my_lcks = &lcks[i * LOCKS_PER_ITER];
+    for (j = 0; j < LOCKS_PER_ITER; j++) {
+      omp_init_lock(&my_lcks[j]);
+    }
+    for (j = 0; j < LOCKS_PER_ITER * 100; j++) {
+      omp_set_lock(&my_lcks[j % LOCKS_PER_ITER]);
+      omp_unset_lock(&my_lcks[j % LOCKS_PER_ITER]);
+    }
+  }
+  // Wait until all repititions are done.  The test is exercising growth of
+  // the global lock pool, which does not shrink when no locks are allocated.
+  {
+    int j;
+    for (j = 0; j < ITERATIONS * LOCKS_PER_ITER; j++) {
+      omp_destroy_lock(&lcks[j]);
+    }
+  }
+
+  return 0;
+}
+
+int main() {
+  // No use repeating this test, since it's exercising a private global pool
+  // which is not reset between test iterations.
+  return test_omp_init_lock();
+}

runtime/test/tasking/omp_taskloop_grainsize.c

@@ -17,7 +17,8 @@
 
 int test_omp_taskloop_grainsize()
 {
-  int i, grainsize, count, tmp_count, result, num_off;
+  int result = 0;
+  int i, grainsize, count, tmp_count, num_off;
   int *tmp, *tids, *tidsArray;
 
   tidsArray = (int *)malloc(sizeof(int) * CFDMAX_SIZE);

runtime/test/worksharing/for/kmp_doacross_check.c

@@ -24,7 +24,7 @@ int main()
   dims.lo = 1;
   dims.up = N-1;
   dims.st = 1;
-  #pragma omp parallel
+  #pragma omp parallel num_threads(4)
   {
     int i, gtid;
     long long vec;