576 lines
16 KiB
C++
Executable File
576 lines
16 KiB
C++
Executable File
// ======================================================================== //
|
|
// Copyright 2009-2014 Intel Corporation //
|
|
// //
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); //
|
|
// you may not use this file except in compliance with the License. //
|
|
// You may obtain a copy of the License at //
|
|
// //
|
|
// http://www.apache.org/licenses/LICENSE-2.0 //
|
|
// //
|
|
// Unless required by applicable law or agreed to in writing, software //
|
|
// distributed under the License is distributed on an "AS IS" BASIS, //
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
|
|
// See the License for the specific language governing permissions and //
|
|
// limitations under the License. //
|
|
// ======================================================================== //
|
|
|
|
#include "barrier.h"
|
|
#include "condition.h"
|
|
|
|
#if defined (__WIN32__)
|
|
|
|
#define WIN32_LEAN_AND_MEAN
|
|
#include <windows.h>
|
|
|
|
namespace embree
|
|
{
|
|
struct BarrierSysImplementation
|
|
{
|
|
__forceinline BarrierSysImplementation ()
|
|
: i(0), enterCount(0), exitCount(0), barrierSize(0)
|
|
{
|
|
events[0] = CreateEvent(NULL, TRUE, FALSE, NULL);
|
|
events[1] = CreateEvent(NULL, TRUE, FALSE, NULL);
|
|
}
|
|
|
|
__forceinline ~BarrierSysImplementation ()
|
|
{
|
|
CloseHandle(events[0]);
|
|
CloseHandle(events[1]);
|
|
}
|
|
|
|
__forceinline void init(size_t N)
|
|
{
|
|
barrierSize = N;
|
|
enterCount = N;
|
|
exitCount = N;
|
|
}
|
|
|
|
__forceinline void wait()
|
|
{
|
|
/* every thread entering the barrier decrements this count */
|
|
size_t i0 = i;
|
|
ssize_t cnt0 = atomic_add(&enterCount,-1);
|
|
|
|
/* all threads except the last one are wait in the barrier */
|
|
if (cnt0 > 1)
|
|
{
|
|
if (WaitForSingleObject(events[i0], INFINITE) != WAIT_OBJECT_0)
|
|
THROW_RUNTIME_ERROR("WaitForSingleObjects failed");
|
|
}
|
|
|
|
/* the last thread starts all threads waiting at the barrier */
|
|
else
|
|
{
|
|
i = 1-i;
|
|
enterCount = barrierSize;
|
|
if (SetEvent(events[i0]) == 0)
|
|
THROW_RUNTIME_ERROR("SetEvent failed");
|
|
}
|
|
|
|
/* every thread leaving the barrier decrements this count */
|
|
ssize_t cnt1 = atomic_add(&exitCount,-1);
|
|
|
|
/* the last thread that left the barrier resets the event again */
|
|
if (cnt1 == 1)
|
|
{
|
|
exitCount = barrierSize;
|
|
if (ResetEvent(events[i0]) == 0)
|
|
THROW_RUNTIME_ERROR("ResetEvent failed");
|
|
}
|
|
}
|
|
|
|
public:
|
|
HANDLE events[2];
|
|
volatile size_t i;
|
|
volatile atomic_t enterCount;
|
|
volatile atomic_t exitCount;
|
|
size_t barrierSize;
|
|
};
|
|
}
|
|
|
|
#else
|
|
|
|
namespace embree
|
|
{
|
|
struct BarrierSysImplementation
|
|
{
|
|
__forceinline BarrierSysImplementation ()
|
|
: count(0), barrierSize(0) {}
|
|
|
|
__forceinline void init(size_t N)
|
|
{
|
|
count = 0;
|
|
barrierSize = N;
|
|
}
|
|
|
|
__forceinline void wait()
|
|
{
|
|
mutex.lock();
|
|
count++;
|
|
|
|
if (count == barrierSize) {
|
|
count = 0;
|
|
cond.broadcast();
|
|
mutex.unlock();
|
|
return;
|
|
}
|
|
|
|
cond.wait(mutex);
|
|
mutex.unlock();
|
|
return;
|
|
}
|
|
|
|
public:
|
|
MutexSys mutex;
|
|
ConditionSys cond;
|
|
volatile size_t count;
|
|
volatile size_t barrierSize;
|
|
};
|
|
}
|
|
|
|
#endif
|
|
|
|
namespace embree
|
|
{
|
|
BarrierSys::BarrierSys () {
|
|
opaque = new BarrierSysImplementation;
|
|
}
|
|
|
|
BarrierSys::~BarrierSys () {
|
|
delete (BarrierSysImplementation*) opaque;
|
|
}
|
|
|
|
void BarrierSys::init(size_t count) {
|
|
((BarrierSysImplementation*) opaque)->init(count);
|
|
}
|
|
|
|
void BarrierSys::wait() {
|
|
((BarrierSysImplementation*) opaque)->wait();
|
|
}
|
|
|
|
LinearBarrierActive::LinearBarrierActive (size_t numThreads_i)
|
|
{
|
|
numThreads = numThreads_i;
|
|
mode = 0;
|
|
flag0 = 0;
|
|
flag1 = 0;
|
|
for (size_t i=0; i<MAX_MIC_THREADS; i++) count0[i] = 0;
|
|
for (size_t i=0; i<MAX_MIC_THREADS; i++) count1[i] = 0;
|
|
}
|
|
|
|
void LinearBarrierActive::init(size_t cntr)
|
|
{
|
|
numThreads = cntr;
|
|
mode = 0;
|
|
flag0 = 0;
|
|
flag1 = 0;
|
|
for (size_t i=0; i<cntr; i++) count0[i] = 0;
|
|
for (size_t i=0; i<cntr; i++) count1[i] = 0;
|
|
}
|
|
|
|
void LinearBarrierActive::wait (const size_t threadIndex, const size_t __threadCount)
|
|
{
|
|
waitForThreads(threadIndex,numThreads);
|
|
}
|
|
|
|
void LinearBarrierActive::waitForThreads(const size_t threadIndex, const size_t threadCount)
|
|
{
|
|
if (mode == 0)
|
|
{
|
|
if (threadIndex == 0)
|
|
{
|
|
for (size_t i=0; i<threadCount; i++)
|
|
count1[i] = 0;
|
|
|
|
for (size_t i=1; i<threadCount; i++)
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(count0[i] == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
mode = 1;
|
|
flag1 = 0;
|
|
__memory_barrier();
|
|
flag0 = 1;
|
|
}
|
|
else
|
|
{
|
|
count0[threadIndex] = 1;
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(flag0 == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (threadIndex == 0)
|
|
{
|
|
for (size_t i=0; i<threadCount; i++)
|
|
count0[i] = 0;
|
|
|
|
for (size_t i=1; i<threadCount; i++)
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(count1[i] == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
|
|
mode = 0;
|
|
flag0 = 0;
|
|
__memory_barrier();
|
|
flag1 = 1;
|
|
}
|
|
else
|
|
{
|
|
count1[threadIndex] = 1;
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(flag1 == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void LinearBarrierActive::syncWithReduction(const size_t threadIndex,
|
|
const size_t threadCount,
|
|
void (* reductionFct)(const size_t currentThreadID,
|
|
const size_t childThreadID,
|
|
void *ptr),
|
|
void *ptr)
|
|
{
|
|
if (mode == 0)
|
|
{
|
|
if (threadIndex == 0)
|
|
{
|
|
for (size_t i=0; i<threadCount; i++)
|
|
count1[i] = 0;
|
|
|
|
for (size_t i=1; i<threadCount; i++)
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(count0[i] == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
(*reductionFct)(threadIndex,i,ptr);
|
|
}
|
|
mode = 1;
|
|
flag1 = 0;
|
|
__memory_barrier();
|
|
flag0 = 1;
|
|
}
|
|
else
|
|
{
|
|
count0[threadIndex] = 1;
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(flag0 == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (threadIndex == 0)
|
|
{
|
|
for (size_t i=0; i<threadCount; i++)
|
|
count0[i] = 0;
|
|
|
|
for (size_t i=1; i<threadCount; i++)
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(count1[i] == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
(*reductionFct)(threadIndex,i,ptr);
|
|
}
|
|
|
|
mode = 0;
|
|
flag0 = 0;
|
|
__memory_barrier();
|
|
flag1 = 1;
|
|
}
|
|
else
|
|
{
|
|
count1[threadIndex] = 1;
|
|
{
|
|
unsigned int wait_cycles = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while (likely(flag1 == 0))
|
|
{
|
|
pause(wait_cycles);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::init()
|
|
{
|
|
*(volatile unsigned int*)&threadState[0][0] = 0;
|
|
*(volatile unsigned int*)&threadState[1][0] = 0;
|
|
mode = 0;
|
|
data[0] = 0;
|
|
__memory_barrier();
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::pause(unsigned int &cycles) {
|
|
__pause_cpu_expfalloff(cycles,MAX_MIC_BARRIER_WAIT_CYCLES);
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::switchModeAndSendRunSignal(const unsigned int m)
|
|
{
|
|
//__memory_barrier();
|
|
mode = 1 - mode;
|
|
__memory_barrier();
|
|
*(volatile unsigned int*)&threadState[m][0] = 0;
|
|
//__memory_barrier();
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::setThreadStateToDone(const unsigned int m, const unsigned int threadID)
|
|
{
|
|
__memory_barrier();
|
|
threadState[m][threadID % 4] = 1;
|
|
__memory_barrier();
|
|
}
|
|
|
|
bool QuadTreeBarrier::CoreSyncData::allThreadsDone(const unsigned int m, const unsigned int orMask) {
|
|
return (*(volatile unsigned int*)&threadState[m][0] | orMask)== 0x01010101;
|
|
}
|
|
|
|
bool QuadTreeBarrier::CoreSyncData::threadDone(const unsigned int m, const unsigned int threadID) {
|
|
return threadState[m][threadID % 4] == 1;
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::waitForAllThreadsOnCore(const unsigned int m)
|
|
{
|
|
unsigned int count = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while(likely(allThreadsDone(m) == false))
|
|
pause(count);
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::waitForAllOtherThreadsOnCore(const unsigned int m, const unsigned int threadID)
|
|
{
|
|
unsigned int count = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
const unsigned int orMask = (unsigned int)1 << ((threadID % 4) * 8);
|
|
while(likely(allThreadsDone(m,orMask) == false))
|
|
pause(count);
|
|
}
|
|
|
|
QuadTreeBarrier::QuadTreeBarrier()
|
|
{
|
|
assert(sizeof(CoreSyncData) == 64);
|
|
for (size_t i=0;i<MAX_MIC_CORES;i++)
|
|
{
|
|
data[i].init();
|
|
}
|
|
}
|
|
|
|
void QuadTreeBarrier::init(size_t cntr) {
|
|
}
|
|
|
|
void QuadTreeBarrier::CoreSyncData::waitForThreadReceivesRunSignal(const unsigned int m, const unsigned int threadID)
|
|
{
|
|
unsigned int count = MIN_MIC_BARRIER_WAIT_CYCLES;
|
|
while(likely(threadDone(m,threadID) == true))
|
|
pause(count);
|
|
}
|
|
|
|
void QuadTreeBarrier::wait(const size_t threadID, const size_t MAX_THREADS_SYNC)
|
|
{
|
|
if (unlikely(MAX_THREADS_SYNC == 1)) return;
|
|
|
|
const unsigned int MAX_CORES_SYNC = MAX_THREADS_SYNC >> 2;
|
|
const unsigned int coreID = threadID >> 2;
|
|
const unsigned int MODE = data[coreID].mode;
|
|
|
|
// Drain store buffer for NGO stores
|
|
//atomic_add((atomic_t*)&data[coreID].data[0],0);
|
|
|
|
//data[coreID].prefetchEx();
|
|
|
|
if (threadID == 0)
|
|
{
|
|
data[0].setThreadStateToDone(MODE,threadID);
|
|
|
|
// == wait for core 0 ==
|
|
data[0].waitForAllThreadsOnCore(MODE);
|
|
|
|
// == wait for (possible) two children cores
|
|
const unsigned int nextCoreID0 = 1;
|
|
const unsigned int nextCoreID1 = 2;
|
|
|
|
data[nextCoreID0].prefetch();
|
|
data[nextCoreID1].prefetch();
|
|
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].waitForAllThreadsOnCore(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].waitForAllThreadsOnCore(MODE);
|
|
|
|
data[nextCoreID0].prefetchEx();
|
|
data[nextCoreID1].prefetchEx();
|
|
|
|
// == run signal to core 0 ==
|
|
data[0].switchModeAndSendRunSignal(MODE);
|
|
|
|
// == propagate run signal to core 1,2 ==
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].switchModeAndSendRunSignal(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].switchModeAndSendRunSignal(MODE);
|
|
}
|
|
else
|
|
{
|
|
|
|
const unsigned int nextCoreID0 = 2*coreID + 1;
|
|
const unsigned int nextCoreID1 = 2*coreID + 2;
|
|
|
|
data[nextCoreID0].prefetch();
|
|
data[nextCoreID1].prefetch();
|
|
|
|
if (threadID % 4 == 0)
|
|
{
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].waitForAllThreadsOnCore(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].waitForAllThreadsOnCore(MODE);
|
|
}
|
|
|
|
data[coreID].setThreadStateToDone(MODE,threadID % 4);
|
|
data[coreID].waitForThreadReceivesRunSignal(MODE,threadID % 4);
|
|
|
|
// == propagte run signal to the two children ==
|
|
|
|
if (threadID % 4 == 0)
|
|
{
|
|
data[nextCoreID0].prefetchEx();
|
|
data[nextCoreID1].prefetchEx();
|
|
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].switchModeAndSendRunSignal(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].switchModeAndSendRunSignal(MODE);
|
|
}
|
|
}
|
|
}
|
|
|
|
void QuadTreeBarrier::syncWithReduction(const size_t threadID,
|
|
const size_t MAX_THREADS_SYNC,
|
|
void (* reductionFct)(const size_t currentThreadID,
|
|
const size_t childThreadID,
|
|
void *ptr),
|
|
void *ptr)
|
|
{
|
|
if (unlikely(MAX_THREADS_SYNC == 1)) return;
|
|
|
|
const unsigned int MAX_CORES_SYNC = MAX_THREADS_SYNC >> 2;
|
|
const unsigned int coreID = threadID >> 2;
|
|
const unsigned int MODE = data[coreID].mode;
|
|
|
|
// Drain store buffer for NGO stores
|
|
//atomic_add((atomic_t*)&data[coreID].data[0],0);
|
|
|
|
data[coreID].prefetchEx();
|
|
|
|
if (threadID == 0)
|
|
{
|
|
|
|
data[0].setThreadStateToDone(MODE,threadID);
|
|
|
|
// == wait for core 0 ==
|
|
data[0].waitForAllThreadsOnCore(MODE);
|
|
|
|
(*reductionFct)(threadID,threadID+1,ptr);
|
|
(*reductionFct)(threadID,threadID+2,ptr);
|
|
(*reductionFct)(threadID,threadID+3,ptr);
|
|
|
|
// == wait for (possible) two children cores
|
|
const unsigned int nextCoreID0 = 1;
|
|
const unsigned int nextCoreID1 = 2;
|
|
const unsigned int nextThreadID0 = nextCoreID0 * 4;
|
|
const unsigned int nextThreadID1 = nextCoreID1 * 4;
|
|
|
|
data[nextCoreID0].prefetch();
|
|
data[nextCoreID1].prefetch();
|
|
|
|
if (nextCoreID0 < MAX_CORES_SYNC)
|
|
{
|
|
data[nextCoreID0].waitForAllThreadsOnCore(MODE);
|
|
(*reductionFct)(threadID,nextThreadID0,ptr);
|
|
}
|
|
|
|
if (nextCoreID1 < MAX_CORES_SYNC)
|
|
{
|
|
data[nextCoreID1].waitForAllThreadsOnCore(MODE);
|
|
(*reductionFct)(threadID,nextThreadID1,ptr);
|
|
}
|
|
|
|
data[nextCoreID0].prefetchEx();
|
|
data[nextCoreID1].prefetchEx();
|
|
|
|
// == run signal to core 0 ==
|
|
data[0].switchModeAndSendRunSignal(MODE);
|
|
|
|
// == propagate run signal to core 1,2 ==
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].switchModeAndSendRunSignal(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].switchModeAndSendRunSignal(MODE);
|
|
}
|
|
else
|
|
{
|
|
|
|
const unsigned int nextCoreID0 = 2*coreID + 1;
|
|
const unsigned int nextCoreID1 = 2*coreID + 2;
|
|
const unsigned int nextThreadID0 = nextCoreID0 * 4;
|
|
const unsigned int nextThreadID1 = nextCoreID1 * 4;
|
|
|
|
data[nextCoreID0].prefetch();
|
|
data[nextCoreID1].prefetch();
|
|
|
|
if (threadID % 4 == 0)
|
|
{
|
|
data[coreID].waitForAllOtherThreadsOnCore(MODE,threadID);
|
|
(*reductionFct)(threadID,threadID+1,ptr);
|
|
(*reductionFct)(threadID,threadID+2,ptr);
|
|
(*reductionFct)(threadID,threadID+3,ptr);
|
|
|
|
if (nextCoreID0 < MAX_CORES_SYNC)
|
|
{
|
|
data[nextCoreID0].waitForAllThreadsOnCore(MODE);
|
|
(*reductionFct)(threadID,nextThreadID0,ptr);
|
|
}
|
|
|
|
|
|
if (nextCoreID1 < MAX_CORES_SYNC)
|
|
{
|
|
data[nextCoreID1].waitForAllThreadsOnCore(MODE);
|
|
(*reductionFct)(threadID,nextThreadID1,ptr);
|
|
}
|
|
}
|
|
|
|
data[coreID].setThreadStateToDone(MODE,threadID % 4);
|
|
|
|
data[coreID].waitForThreadReceivesRunSignal(MODE,threadID % 4);
|
|
|
|
// == propagte run signal to the two children ==
|
|
|
|
if (threadID % 4 == 0)
|
|
{
|
|
data[nextCoreID0].prefetchEx();
|
|
data[nextCoreID1].prefetchEx();
|
|
|
|
if (nextCoreID0 < MAX_CORES_SYNC) data[nextCoreID0].switchModeAndSendRunSignal(MODE);
|
|
if (nextCoreID1 < MAX_CORES_SYNC) data[nextCoreID1].switchModeAndSendRunSignal(MODE);
|
|
}
|
|
}
|
|
}
|
|
}
|