From d5472758458eee5d687388734f08de39c42d3faa Mon Sep 17 00:00:00 2001 From: Felix Weiglhofer Date: Fri, 29 May 2020 18:48:39 +0200 Subject: [PATCH] GPU: Parallize track merger resolve kernel. --- GPU/Common/GPUCommonMath.h | 36 +- GPU/Common/GPUDefGPUParameters.h | 23 +- .../Base/GPUReconstructionKernels.h | 5 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 17 +- GPU/GPUTracking/Global/GPUChainTracking.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 317 ++++++++++++------ GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 16 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 26 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 9 +- 9 files changed, 321 insertions(+), 129 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index f925cd6fbfb4f..309ebc38c0a46 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -73,6 +73,13 @@ class GPUCommonMath { return GPUCommonMath::AtomicExchInt(addr, val); } + + template + GPUdi() static T AtomicCAS(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T cmp, T val) + { + return GPUCommonMath::AtomicCASInt(addr, cmp, val); + } + template GPUdi() static T AtomicAdd(GPUglobalref() GPUgeneric() GPUAtomic(T) * addr, T val) { @@ -138,6 +145,8 @@ class GPUCommonMath template GPUd() static unsigned int AtomicExchInt(S* addr, T val); template + GPUd() static T AtomicCASInt(S* addr, T cmp, T val); + template GPUd() static unsigned int AtomicAddInt(S* addr, T val); template GPUd() static void AtomicMaxInt(S* addr, T val); @@ -367,9 +376,10 @@ GPUhdi() float GPUCommonMath::Copysign(float x, float y) #endif // GPUCA_GPUCODE } -#ifndef GPUCA_GPUCODE +#if !defined(GPUCA_GPUCODE) && defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-value" // GCC BUG in omp atomic capture gives false warning +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif template @@ -394,6 +404,28 @@ GPUdi() unsigned int GPUCommonMath::AtomicExchInt(S* addr, T val) #endif // GPUCA_GPUCODE } +template +GPUdi() T GPUCommonMath::AtomicCASInt(S* addr, T cmp, T val) +{ +#if defined(GPUCA_GPUCODE) && defined(__OPENCLCPP__) && (!defined(__clang__) || defined(GPUCA_OPENCL_CPP_CLANG_C11_ATOMICS)) + return ::atomic_compare_exchange(addr, cmp, val); +#elif defined(GPUCA_GPUCODE) && defined(__OPENCL__) + return ::atomic_cmpxchg(addr, cmp, val); +#elif defined(GPUCA_GPUCODE) && (defined(__CUDACC__) || defined(__HIPCC__)) + return ::atomicCAS(addr, cmp, val); +#else + T old; +#ifdef WITH_OPENMP +#pragma omp atomic capture +#endif + { + old = *addr; + *addr = (old == cmp) ? val : old; + } + return old; +#endif // GPUCA_GPUCODE +} + template GPUdi() unsigned int GPUCommonMath::AtomicAddInt(S* addr, T val) { @@ -456,7 +488,7 @@ GPUdi() void GPUCommonMath::AtomicMinInt(S* addr, T val) #endif // GPUCA_GPUCODE } -#ifndef GPUCA_GPUCODE +#if !defined(GPUCA_GPUCODE) && defined(__GNUC__) && !defined(__clang__) #pragma GCC diagnostic pop #endif diff --git a/GPU/Common/GPUDefGPUParameters.h b/GPU/Common/GPUDefGPUParameters.h index 0ca008fb373f4..55c5937cace27 100644 --- a/GPU/Common/GPUDefGPUParameters.h +++ b/GPU/Common/GPUDefGPUParameters.h @@ -45,7 +45,10 @@ #define GPUCA_LB_GPUTPCGMMergerSliceRefit 256 #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256 #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256 #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256 @@ -90,7 +93,10 @@ #define GPUCA_LB_GPUTPCGMMergerSliceRefit 64 #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256 #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256 #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256, 2 @@ -186,8 +192,17 @@ #ifndef GPUCA_LB_GPUTPCGMMergerUnpackGlobal #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 #endif - #ifndef GPUCA_LB_GPUTPCGMMergerResolve - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step0 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step1 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step2 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step3 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #endif #ifndef GPUCA_LB_GPUTPCGMMergerClearLinks #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 diff --git a/GPU/GPUTracking/Base/GPUReconstructionKernels.h b/GPU/GPUTracking/Base/GPUReconstructionKernels.h index e55f6147cee16..398470255d9e7 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionKernels.h +++ b/GPU/GPUTracking/Base/GPUReconstructionKernels.h @@ -33,7 +33,10 @@ GPUCA_KRNL_LB((GPUTPCGMMergerUnpackResetIds ), (simple), (, int GPUCA_KRNL_LB((GPUTPCGMMergerSliceRefit ), (simple), (, int iSlice), (, iSlice)) GPUCA_KRNL_LB((GPUTPCGMMergerUnpackGlobal ), (simple), (, int iSlice), (, iSlice)) GPUCA_KRNL(( GPUTPCGMMergerUnpackSaveNumber ), (simple), (, int id), (, id)) -GPUCA_KRNL_LB((GPUTPCGMMergerResolve ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll)) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step0 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step1 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step2 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step3 ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll)) GPUCA_KRNL_LB((GPUTPCGMMergerClearLinks ), (simple), (, char nOutput), (, nOutput)) GPUCA_KRNL_LB((GPUTPCGMMergerMergeWithinPrepare ), (simple), (), ()) GPUCA_KRNL_LB((GPUTPCGMMergerMergeSlicesPrepare ), (simple), (, int border0, int border1, char useOrigTrackParam), (, border0, border1, useOrigTrackParam)) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 264e0f82f0e29..c8cde6dd9bb5a 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1668,6 +1668,15 @@ void GPUChainTracking::RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, mRec->ReturnVolatileDeviceMemory(); } +void GPUChainTracking::RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType) +{ + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + // TODO: Determine number of blocks from block size to fully occupy gpu + runKernel(GetGridBlk(BlockCount() * 4, 0, deviceType), krnlRunRangeNone, krnlEventNone, useOrigTrackParam, mergeAll); +} + int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) { if (GetDeviceProcessingSettings().debugLevel >= 6 && GetDeviceProcessingSettings().comparableDebutOutput && param().rec.mergerReadFromTrackerDirectly) { @@ -1727,22 +1736,22 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 0237e5e4d8b8b..a2dc9b183ef72 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -251,6 +251,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int stage, bool doGPU, int lane); std::pair TPCClusterizerDecodeZSCount(unsigned int iSlice, unsigned int minTime, unsigned int maxTime); void RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, char mergeMode, GPUReconstruction::krnlDeviceType deviceType); + void RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType); std::atomic_flag mLockAtomic = ATOMIC_FLAG_INIT; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 0513119b73532..bbae806068e9a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -229,6 +229,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mBorderMemory, 2 * mNMaxSliceTracks); computePointerWithAlignment(mem, mBorderRangeMemory, 2 * mNMaxSliceTracks); computePointerWithAlignment(mem, mTrackLinks, mNMaxSliceTracks); + computePointerWithAlignment(mem, mTrackCCRoots, mNMaxSliceTracks); size_t tmpSize = CAMath::Max(CAMath::Max(mNMaxSingleSliceTracks, 1) * NSLICES * sizeof(int), CAMath::nextMultipleOf<4>(mNMaxTracks) * sizeof(int) + mNMaxClusters * sizeof(unsigned int)); computePointerWithAlignment(mem, mTmpMem, (tmpSize + sizeof(*mTmpMem) - 1) / sizeof(*mTmpMem)); @@ -903,7 +904,73 @@ GPUd() void GPUTPCGMMerger::MergeSlicesPrepare(int nBlocks, int nThreads, int iB MakeBorderTracks((nBlocks + 1) >> 1, nThreads, iBlock >> 1, iThread, border, b, n, useOrigTrackParam); } -GPUd() void GPUTPCGMMerger::ResolveMergeSlices(int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll) +GPUdi() void GPUTPCGMMerger::setBlockRange(int elems, int nBlocks, int iBlock, int& start, int& end) +{ + start = (elems + nBlocks - 1) / nBlocks * iBlock; + end = (elems + nBlocks - 1) / nBlocks * (iBlock + 1); + end = CAMath::Min(elems, end); +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsSetup(int nBlocks, int nThreads, int iBlock, int iThread) +{ + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int i = start + iThread; i < end; i += nThreads) { + mTrackCCRoots[i] = i; + } +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsHook(int nBlocks, int nThreads, int iBlock, int iThread) +{ + // Compute connected components in parallel, step 1. Source: Adaptive Work-Efficient Connected Components onthe GPU, Sutton et al, 2016 (https://arxiv.org/pdf/1612.01178.pdf) + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int itr = start + iThread; itr < end; itr += nThreads) { + int u = itr; + int v = mTrackLinks[u]; + if (v < 0) { + continue; + } + while (true) { + u = mTrackCCRoots[u]; + v = mTrackCCRoots[v]; + if (u == v) { + break; + } + int h = CAMath::Max(u, v); + int l = CAMath::Min(u, v); + + int old = CAMath::AtomicCAS(&mTrackCCRoots[h], h, l); + if (old == h) { + break; + } + + u = mTrackCCRoots[h]; + v = l; + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsMultiJump(int nBlocks, int nThreads, int iBlock, int iThread) +{ + // Compute connected components in parallel, step 2. + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int itr = start + iThread; itr < end; itr += nThreads) { + int root = itr; + int next = mTrackCCRoots[root]; + if (root == next) { + continue; + } + do { + root = next; + next = mTrackCCRoots[next]; + } while (root != next); + mTrackCCRoots[itr] = root; + } +} + +GPUd() void GPUTPCGMMerger::ResolveMergeSlices(GPUResolveSharedMemory& smem, int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll) { if (!mergeAll) { /*int neighborType = useOrigTrackParam ? 1 : 0; @@ -935,137 +1002,171 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSlices(int nBlocks, int nThreads, int iB newTrack2.SetPrevNeighbour( itr, neighborType );*/ } - for (int itr = 0; itr < SliceTrackInfoLocalTotal(); itr++) { - int itr2 = mTrackLinks[itr]; - if (itr2 < 0) { - continue; + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + + for (int baseIdx = 0; baseIdx < SliceTrackInfoLocalTotal(); baseIdx += nThreads) { + int itr = baseIdx + iThread; + bool inRange = itr < SliceTrackInfoLocalTotal(); + + int itr2 = -1; + if (inRange) { + itr2 = mTrackLinks[itr]; + } + + bool resolveSlice = (itr2 > -1); + if (resolveSlice) { + int root = mTrackCCRoots[itr]; + resolveSlice &= (start <= root) && (root < end); } - GPUTPCGMSliceTrack* track1 = &mSliceTrackInfos[itr]; - GPUTPCGMSliceTrack* track2 = &mSliceTrackInfos[itr2]; - GPUTPCGMSliceTrack* track1Base = track1; - GPUTPCGMSliceTrack* track2Base = track2; - bool sameSegment = CAMath::Abs(track1->NClusters() > track2->NClusters() ? track1->QPt() : track2->QPt()) < 2 || track1->QPt() * track2->QPt() > 0; - // GPUInfo("\nMerge %d with %d - same segment %d", itr, itr2, (int) sameSegment); - // PrintMergeGraph(track1, std::cout); - // PrintMergeGraph(track2, std::cout); + short smemIdx = work_group_scan_inclusive_add(short(resolveSlice)); - while (track2->PrevSegmentNeighbour() >= 0) { - track2 = &mSliceTrackInfos[track2->PrevSegmentNeighbour()]; + if (resolveSlice) { + smem.iTrack1[smemIdx - 1] = itr; + smem.iTrack2[smemIdx - 1] = itr2; } - if (sameSegment) { - if (track1 == track2) { - continue; + GPUbarrier(); + + if (iThread < nThreads - 1) { + continue; + } + + const int nSlices = smemIdx; + + for (int i = 0; i < nSlices; i++) { + itr = smem.iTrack1[i]; + itr2 = smem.iTrack2[i]; + + GPUTPCGMSliceTrack* track1 = &mSliceTrackInfos[itr]; + GPUTPCGMSliceTrack* track2 = &mSliceTrackInfos[itr2]; + GPUTPCGMSliceTrack* track1Base = track1; + GPUTPCGMSliceTrack* track2Base = track2; + + bool sameSegment = CAMath::Abs(track1->NClusters() > track2->NClusters() ? track1->QPt() : track2->QPt()) < 2 || track1->QPt() * track2->QPt() > 0; + // GPUInfo("\nMerge %d with %d - same segment %d", itr, itr2, (int) sameSegment); + // PrintMergeGraph(track1, std::cout); + // PrintMergeGraph(track2, std::cout); + + while (track2->PrevSegmentNeighbour() >= 0) { + track2 = &mSliceTrackInfos[track2->PrevSegmentNeighbour()]; } - while (track1->PrevSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + if (sameSegment) { if (track1 == track2) { - goto NextTrack; + continue; } - } - GPUCommonAlgorithm::swap(track1, track1Base); - for (int k = 0; k < 2; k++) { - GPUTPCGMSliceTrack* tmp = track1Base; - while (tmp->Neighbour(k) >= 0) { - tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; - if (tmp == track2) { + while (track1->PrevSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + if (track1 == track2) { goto NextTrack; } } - } - - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; - if (track1 == track2) { - goto NextTrack; + GPUCommonAlgorithm::swap(track1, track1Base); + for (int k = 0; k < 2; k++) { + GPUTPCGMSliceTrack* tmp = track1Base; + while (tmp->Neighbour(k) >= 0) { + tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; + if (tmp == track2) { + goto NextTrack; + } + } } - } - } else { - while (track1->PrevSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; - } - if (track1 == track2) { - continue; - } - for (int k = 0; k < 2; k++) { - GPUTPCGMSliceTrack* tmp = track1; - while (tmp->Neighbour(k) >= 0) { - tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; - if (tmp == track2) { + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + if (track1 == track2) { goto NextTrack; } } - } + } else { + while (track1->PrevSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + } - float z1min, z1max, z2min, z2max; - z1min = track1->MinClusterZT(); - z1max = track1->MaxClusterZT(); - z2min = track2->MinClusterZT(); - z2max = track2->MaxClusterZT(); - if (track1 != track1Base) { - z1min = CAMath::Min(z1min, track1Base->MinClusterZT()); - z1max = CAMath::Max(z1max, track1Base->MaxClusterZT()); - } - if (track2 != track2Base) { - z2min = CAMath::Min(z2min, track2Base->MinClusterZT()); - z2max = CAMath::Max(z2max, track2Base->MaxClusterZT()); - } - bool goUp = z2max - z1min > z1max - z2min; + if (track1 == track2) { + continue; + } + for (int k = 0; k < 2; k++) { + GPUTPCGMSliceTrack* tmp = track1; + while (tmp->Neighbour(k) >= 0) { + tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; + if (tmp == track2) { + goto NextTrack; + } + } + } - if (track1->Neighbour(goUp) < 0 && track2->Neighbour(!goUp) < 0) { - track1->SetNeighbor(track2 - mSliceTrackInfos, goUp); - track2->SetNeighbor(track1 - mSliceTrackInfos, !goUp); - // GPUInfo("Result (simple neighbor)"); - // PrintMergeGraph(track1, std::cout); - continue; - } else if (track1->Neighbour(goUp) < 0) { - track2 = &mSliceTrackInfos[track2->Neighbour(!goUp)]; - GPUCommonAlgorithm::swap(track1, track2); - } else if (track2->Neighbour(!goUp) < 0) { - track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; - } else { // Both would work, but we use the simpler one - track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + float z1min, z1max, z2min, z2max; + z1min = track1->MinClusterZT(); + z1max = track1->MaxClusterZT(); + z2min = track2->MinClusterZT(); + z2max = track2->MaxClusterZT(); + if (track1 != track1Base) { + z1min = CAMath::Min(z1min, track1Base->MinClusterZT()); + z1max = CAMath::Max(z1max, track1Base->MaxClusterZT()); + } + if (track2 != track2Base) { + z2min = CAMath::Min(z2min, track2Base->MinClusterZT()); + z2max = CAMath::Max(z2max, track2Base->MaxClusterZT()); + } + bool goUp = z2max - z1min > z1max - z2min; + + if (track1->Neighbour(goUp) < 0 && track2->Neighbour(!goUp) < 0) { + track1->SetNeighbor(track2 - mSliceTrackInfos, goUp); + track2->SetNeighbor(track1 - mSliceTrackInfos, !goUp); + // GPUInfo("Result (simple neighbor)"); + // PrintMergeGraph(track1, std::cout); + continue; + } else if (track1->Neighbour(goUp) < 0) { + track2 = &mSliceTrackInfos[track2->Neighbour(!goUp)]; + GPUCommonAlgorithm::swap(track1, track2); + } else if (track2->Neighbour(!goUp) < 0) { + track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + } else { // Both would work, but we use the simpler one + track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + } + track1Base = track1; } - track1Base = track1; - } - track2Base = track2; - if (!sameSegment) { - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + track2Base = track2; + if (!sameSegment) { + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + } } - } - track1->SetNextSegmentNeighbour(track2 - mSliceTrackInfos); - track2->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); - for (int k = 0; k < 2; k++) { - track1 = track1Base; - track2 = track2Base; - while (track2->Neighbour(k) >= 0) { - if (track1->Neighbour(k) >= 0) { - GPUTPCGMSliceTrack* track1new = &mSliceTrackInfos[track1->Neighbour(k)]; - GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; - track2->SetNeighbor(-1, k); - track2new->SetNeighbor(-1, k ^ 1); - track1 = track1new; - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + track1->SetNextSegmentNeighbour(track2 - mSliceTrackInfos); + track2->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); + // k = 0: Merge right side + // k = 1: Merge left side + for (int k = 0; k < 2; k++) { + track1 = track1Base; + track2 = track2Base; + while (track2->Neighbour(k) >= 0) { + if (track1->Neighbour(k) >= 0) { + GPUTPCGMSliceTrack* track1new = &mSliceTrackInfos[track1->Neighbour(k)]; + GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; + track2->SetNeighbor(-1, k); + track2new->SetNeighbor(-1, k ^ 1); + track1 = track1new; + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + } + track1->SetNextSegmentNeighbour(track2new - mSliceTrackInfos); + track2new->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); + track1 = track1new; + track2 = track2new; + } else { + GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; + track1->SetNeighbor(track2->Neighbour(k), k); + track2->SetNeighbor(-1, k); + track2new->SetNeighbor(track1 - mSliceTrackInfos, k ^ 1); } - track1->SetNextSegmentNeighbour(track2new - mSliceTrackInfos); - track2new->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); - track1 = track1new; - track2 = track2new; - } else { - GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; - track1->SetNeighbor(track2->Neighbour(k), k); - track2->SetNeighbor(-1, k); - track2new->SetNeighbor(track1 - mSliceTrackInfos, k ^ 1); } } + // GPUInfo("Result"); + // PrintMergeGraph(track1, std::cout); + NextTrack:; } - // GPUInfo("Result"); - // PrintMergeGraph(track1, std::cout); - NextTrack:; } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 6b01ecf0183e8..f2b4bb6691946 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -22,6 +22,7 @@ #include "GPUCommonDef.h" #include "GPUProcessor.h" #include "GPUTPCGMMergerTypes.h" +#include "GPUGeneralKernels.h" #if !defined(GPUCA_GPUCODE) #include @@ -79,6 +80,11 @@ class GPUTPCGMMerger : public GPUProcessor unsigned char leg; }; + struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { + int iTrack1[GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCGMMergerResolve_step3)]; + int iTrack2[GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCGMMergerResolve_step3)]; + }; + void InitializeProcessor(); void RegisterMemoryAllocation(); void SetMaxData(const GPUTrackingInOutPointers& io); @@ -149,7 +155,10 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void Finalize0(int nBlocks, int nThreads, int iBlock, int iThread); GPUd() void Finalize1(int nBlocks, int nThreads, int iBlock, int iThread); GPUd() void Finalize2(int nBlocks, int nThreads, int iBlock, int iThread); - GPUd() void ResolveMergeSlices(int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll); + GPUd() void ResolveFindConnectedComponentsSetup(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveFindConnectedComponentsHook(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveFindConnectedComponentsMultiJump(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveMergeSlices(GPUResolveSharedMemory& smem, int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll); #ifndef GPUCA_GPUCODE void DumpSliceTracks(std::ostream& out); @@ -185,12 +194,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUdi() int SliceTrackInfoLocalTotal() { return mSliceTrackInfoIndex[NSLICES]; } GPUdi() int SliceTrackInfoTotal() { return mSliceTrackInfoIndex[2 * NSLICES]; } + GPUdi() void setBlockRange(int elems, int nBlocks, int iBlock, int& start, int& end); + int mNextSliceInd[NSLICES]; int mPrevSliceInd[NSLICES]; const GPUTPCSliceOutput* mkSlices[NSLICES]; //* array of input slice tracks int* mTrackLinks; + int* mTrackCCRoots; // root of the connected component of this track unsigned int mNMaxSliceTracks; // maximum number of incoming slice tracks unsigned int mNMaxTracks; // maximum number of output tracks @@ -201,7 +213,7 @@ class GPUTPCGMMerger : public GPUProcessor unsigned short mMemoryResMemory; unsigned short mMemoryResOutput; - int mNClusters; // Total number of incoming clusters (from slice tracks) + int mNClusters; // Total number of incoming clusters (from slice tracks) GPUTPCGMMergedTrack* mOutputTracks; //* array of output merged tracks GPUTPCGMSliceTrack* mSliceTrackInfos; //* additional information for slice tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 508a169cca26d..ec8a4456b7adf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -12,6 +12,7 @@ /// \author David Rohr #include "GPUTPCGMMergerGPU.h" +#include "GPUCommonAlgorithm.h" #if defined(WITH_OPENMP) && !defined(GPUCA_GPUCODE) #include "GPUReconstruction.h" #endif @@ -66,12 +67,27 @@ GPUdii() void GPUTPCGMMergerUnpackSaveNumber::Thread<0>(int nBlocks, int nThread } template <> -GPUdii() void GPUTPCGMMergerResolve::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger, char useOrigTrackParam, char mergeAll) +GPUdii() void GPUTPCGMMergerResolve::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) { - if (iThread || iBlock) { - return; - } - merger.ResolveMergeSlices(nBlocks, nThreads, iBlock, iThread, useOrigTrackParam, mergeAll); + merger.ResolveFindConnectedComponentsSetup(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<1>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) +{ + merger.ResolveFindConnectedComponentsHook(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<2>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) +{ + merger.ResolveFindConnectedComponentsMultiJump(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<3>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger, char useOrigTrackParam, char mergeAll) +{ + merger.ResolveMergeSlices(smem, nBlocks, nThreads, iBlock, iThread, useOrigTrackParam, mergeAll); } template <> diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 3906e66f0e62e..bb2cde4f68060 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -91,9 +91,12 @@ class GPUTPCGMMergerUnpackResetIds : public GPUTPCGMMergerGeneral class GPUTPCGMMergerResolve : public GPUTPCGMMergerGeneral { public: -#if !defined(GPUCA_ALIROOT_LIB) || !defined(GPUCA_GPUCODE) - template - GPUd() static void Thread(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, char useOrigTrackParam, char mergeAll); + struct GPUSharedMemory : public GPUTPCGMMerger::GPUResolveSharedMemory { + }; + +#if !defined(GPUCA_ALIROOT_LIB) + template + GPUd() static void Thread(int nBlocks, int nThreads, int iBlock, int iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); #endif };