Skip to content

Commit d547275

Browse files
fweigfweig
committed
GPU: Parallize track merger resolve kernel.
1 parent 2ff1e48 commit d547275

9 files changed

Lines changed: 321 additions & 129 deletions

File tree

‎GPU/Common/GPUCommonMath.h‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,13 @@ class GPUCommonMath
7373
{
7474
return GPUCommonMath::AtomicExchInt(addr, val);
7575
}
76+
77+
template <class T>
78+
GPUdi() static T AtomicCAS(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T cmp, T val)
79+
{
80+
return GPUCommonMath::AtomicCASInt(addr, cmp, val);
81+
}
82+
7683
template <class T>
7784
GPUdi() static T AtomicAdd(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val)
7885
{
@@ -138,6 +145,8 @@ class GPUCommonMath
138145
template <class S, class T>
139146
GPUd() static unsigned int AtomicExchInt(S* addr, T val);
140147
template <class S, class T>
148+
GPUd() static T AtomicCASInt(S* addr, T cmp, T val);
149+
template <class S, class T>
141150
GPUd() static unsigned int AtomicAddInt(S* addr, T val);
142151
template <class S, class T>
143152
GPUd() static void AtomicMaxInt(S* addr, T val);
@@ -367,9 +376,10 @@ GPUhdi() float GPUCommonMath::Copysign(float x, float y)
367376
#endif // GPUCA_GPUCODE
368377
}
369378

370-
#ifndef GPUCA_GPUCODE
379+
#if !defined(GPUCA_GPUCODE) && defined(__GNUC__) && !defined(__clang__)
371380
#pragma GCC diagnostic push
372381
#pragma GCC diagnostic ignored "-Wunused-value" // GCC BUG in omp atomic capture gives false warning
382+
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
373383
#endif
374384

375385
template <class S, class T>
@@ -394,6 +404,28 @@ GPUdi() unsigned int GPUCommonMath::AtomicExchInt(S* addr, T val)
394404
#endif // GPUCA_GPUCODE
395405
}
396406

407+
template <class S, class T>
408+
GPUdi() T GPUCommonMath::AtomicCASInt(S* addr, T cmp, T val)
409+
{
410+
#if defined(GPUCA_GPUCODE) && defined(__OPENCLCPP__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CPP_CLANG_C11_ATOMICS))
411+
return ::atomic_compare_exchange(addr, cmp, val);
412+
#elif defined(GPUCA_GPUCODE) && defined(__OPENCL__)
413+
return ::atomic_cmpxchg(addr, cmp, val);
414+
#elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__))
415+
return ::atomicCAS(addr, cmp, val);
416+
#else
417+
T old;
418+
#ifdef WITH_OPENMP
419+
#pragma omp atomic capture
420+
#endif
421+
{
422+
old = *addr;
423+
*addr = (old == cmp) ? val : old;
424+
}
425+
return old;
426+
#endif // GPUCA_GPUCODE
427+
}
428+
397429
template <class S, class T>
398430
GPUdi() unsigned int GPUCommonMath::AtomicAddInt(S* addr, T val)
399431
{
@@ -456,7 +488,7 @@ GPUdi() void GPUCommonMath::AtomicMinInt(S* addr, T val)
456488
#endif // GPUCA_GPUCODE
457489
}
458490

459-
#ifndef GPUCA_GPUCODE
491+
#if !defined(GPUCA_GPUCODE) && defined(__GNUC__) && !defined(__clang__)
460492
#pragma GCC diagnostic pop
461493
#endif
462494

‎GPU/Common/GPUDefGPUParameters.h‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@
4545
#define GPUCA_LB_GPUTPCGMMergerSliceRefit 256
4646
#define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256
4747
#define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
48-
#define GPUCA_LB_GPUTPCGMMergerResolve 256
48+
#define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
49+
#define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
50+
#define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
51+
#define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
4952
#define GPUCA_LB_GPUTPCGMMergerClearLinks 256
5053
#define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256
5154
#define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256
@@ -90,7 +93,10 @@
9093
#define GPUCA_LB_GPUTPCGMMergerSliceRefit 64
9194
#define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256
9295
#define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
93-
#define GPUCA_LB_GPUTPCGMMergerResolve 256
96+
#define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
97+
#define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
98+
#define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
99+
#define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
94100
#define GPUCA_LB_GPUTPCGMMergerClearLinks 256
95101
#define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256
96102
#define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256, 2
@@ -186,8 +192,17 @@
186192
#ifndef GPUCA_LB_GPUTPCGMMergerUnpackGlobal
187193
#define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
188194
#endif
189-
#ifndef GPUCA_LB_GPUTPCGMMergerResolve
190-
#define GPUCA_LB_GPUTPCGMMergerResolve 256
195+
#ifndef GPUCA_LB_GPUTPCGMMergerResolve_step0
196+
#define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
197+
#endif
198+
#ifndef GPUCA_LB_GPUTPCGMMergerResolve_step1
199+
#define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
200+
#endif
201+
#ifndef GPUCA_LB_GPUTPCGMMergerResolve_step2
202+
#define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
203+
#endif
204+
#ifndef GPUCA_LB_GPUTPCGMMergerResolve_step3
205+
#define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
191206
#endif
192207
#ifndef GPUCA_LB_GPUTPCGMMergerClearLinks
193208
#define GPUCA_LB_GPUTPCGMMergerClearLinks 256

‎GPU/GPUTracking/Base/GPUReconstructionKernels.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ GPUCA_KRNL_LB((GPUTPCGMMergerUnpackResetIds ), (simple), (, int
3333
GPUCA_KRNL_LB((GPUTPCGMMergerSliceRefit ), (simple), (, int iSlice), (, iSlice))
3434
GPUCA_KRNL_LB((GPUTPCGMMergerUnpackGlobal ), (simple), (, int iSlice), (, iSlice))
3535
GPUCA_KRNL(( GPUTPCGMMergerUnpackSaveNumber ), (simple), (, int id), (, id))
36-
GPUCA_KRNL_LB((GPUTPCGMMergerResolve ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll))
36+
GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step0 ), (simple), (), ())
37+
GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step1 ), (simple), (), ())
38+
GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step2 ), (simple), (), ())
39+
GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step3 ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll))
3740
GPUCA_KRNL_LB((GPUTPCGMMergerClearLinks ), (simple), (, char nOutput), (, nOutput))
3841
GPUCA_KRNL_LB((GPUTPCGMMergerMergeWithinPrepare ), (simple), (), ())
3942
GPUCA_KRNL_LB((GPUTPCGMMergerMergeSlicesPrepare ), (simple), (, int border0, int border1, char useOrigTrackParam), (, border0, border1, useOrigTrackParam))

‎GPU/GPUTracking/Global/GPUChainTracking.cxx‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,6 +1668,15 @@ void GPUChainTracking::RunTPCTrackingMerger_MergeBorderTracks(char withinSlice,
16681668
mRec->ReturnVolatileDeviceMemory();
16691669
}
16701670

1671+
void GPUChainTracking::RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType)
1672+
{
1673+
runKernel<GPUTPCGMMergerResolve, 0>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
1674+
runKernel<GPUTPCGMMergerResolve, 1>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
1675+
runKernel<GPUTPCGMMergerResolve, 2>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
1676+
// TODO: Determine number of blocks from block size to fully occupy gpu
1677+
runKernel<GPUTPCGMMergerResolve, 3>(GetGridBlk(BlockCount() * 4, 0, deviceType), krnlRunRangeNone, krnlEventNone, useOrigTrackParam, mergeAll);
1678+
}
1679+
16711680
int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
16721681
{
16731682
if (GetDeviceProcessingSettings().debugLevel >= 6 && GetDeviceProcessingSettings().comparableDebutOutput && param().rec.mergerReadFromTrackerDirectly) {
@@ -1727,22 +1736,22 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
17271736
runKernel<GPUTPCGMMergerMergeWithinPrepare>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone);
17281737

17291738
RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType);
1730-
runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1);
1739+
RunTPCTrackingMerger_Resolve(0, 1, deviceType);
17311740
DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile);
17321741

17331742
runKernel<GPUTPCGMMergerClearLinks>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0);
17341743
runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
17351744
runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0);
17361745
RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType);
1737-
runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0);
1746+
RunTPCTrackingMerger_Resolve(0, 1, deviceType);
17381747
runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
17391748
runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0);
17401749
RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType);
1741-
runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0);
1750+
RunTPCTrackingMerger_Resolve(0, 1, deviceType);
17421751
runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
17431752
runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1);
17441753
RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType);
1745-
runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0);
1754+
RunTPCTrackingMerger_Resolve(0, 1, deviceType);
17461755
DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile);
17471756

17481757
runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));

‎GPU/GPUTracking/Global/GPUChainTracking.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega
251251
void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int stage, bool doGPU, int lane);
252252
std::pair<unsigned int, unsigned int> TPCClusterizerDecodeZSCount(unsigned int iSlice, unsigned int minTime, unsigned int maxTime);
253253
void RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, char mergeMode, GPUReconstruction::krnlDeviceType deviceType);
254+
void RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType);
254255

255256
std::atomic_flag mLockAtomic = ATOMIC_FLAG_INIT;
256257

0 commit comments

Comments
 (0)