diff --git a/GPU/Common/GPUDefGPUParameters.h b/GPU/Common/GPUDefGPUParameters.h index 0ca008fb373f4..55c5937cace27 100644 --- a/GPU/Common/GPUDefGPUParameters.h +++ b/GPU/Common/GPUDefGPUParameters.h @@ -45,7 +45,10 @@ #define GPUCA_LB_GPUTPCGMMergerSliceRefit 256 #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256 #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256 #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256 @@ -90,7 +93,10 @@ #define GPUCA_LB_GPUTPCGMMergerSliceRefit 64 #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256 #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256 #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256, 2 @@ -186,8 +192,17 @@ #ifndef GPUCA_LB_GPUTPCGMMergerUnpackGlobal #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256 #endif - #ifndef GPUCA_LB_GPUTPCGMMergerResolve - #define GPUCA_LB_GPUTPCGMMergerResolve 256 + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step0 + #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step1 + #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step2 + #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256 + #endif + #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step3 + #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256 #endif #ifndef GPUCA_LB_GPUTPCGMMergerClearLinks #define GPUCA_LB_GPUTPCGMMergerClearLinks 256 diff --git a/GPU/GPUTracking/Base/GPUReconstructionKernels.h b/GPU/GPUTracking/Base/GPUReconstructionKernels.h index e55f6147cee16..398470255d9e7 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionKernels.h +++ b/GPU/GPUTracking/Base/GPUReconstructionKernels.h @@ -33,7 +33,10 @@ GPUCA_KRNL_LB((GPUTPCGMMergerUnpackResetIds ), (simple), (, int GPUCA_KRNL_LB((GPUTPCGMMergerSliceRefit ), (simple), (, int iSlice), (, iSlice)) GPUCA_KRNL_LB((GPUTPCGMMergerUnpackGlobal ), (simple), (, int iSlice), (, iSlice)) GPUCA_KRNL(( GPUTPCGMMergerUnpackSaveNumber ), (simple), (, int id), (, id)) -GPUCA_KRNL_LB((GPUTPCGMMergerResolve ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll)) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step0 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step1 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step2 ), (simple), (), ()) +GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step3 ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll)) GPUCA_KRNL_LB((GPUTPCGMMergerClearLinks ), (simple), (, char nOutput), (, nOutput)) GPUCA_KRNL_LB((GPUTPCGMMergerMergeWithinPrepare ), (simple), (), ()) GPUCA_KRNL_LB((GPUTPCGMMergerMergeSlicesPrepare ), (simple), (, int border0, int border1, char useOrigTrackParam), (, border0, border1, useOrigTrackParam)) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 0757e83ba4b9d..cb5c35b033edd 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1643,6 +1643,15 @@ void GPUChainTracking::RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, mRec->ReturnVolatileDeviceMemory(); } +void GPUChainTracking::RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType) +{ + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + runKernel(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone); + // TODO: Determine number of blocks from block size to fully occupy gpu + runKernel(GetGridBlk(BlockCount() * 4, 0, deviceType), krnlRunRangeNone, krnlEventNone, useOrigTrackParam, mergeAll); +} + int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) { if (GetDeviceProcessingSettings().debugLevel >= 6 && GetDeviceProcessingSettings().comparableDebutOutput && param().rec.mergerReadFromTrackerDirectly) { @@ -1701,22 +1710,22 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone); RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile); runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType); - runKernel(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0); + RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile); runKernel({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter())); diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 01f2d4f56e098..a48a7f97758f3 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -250,6 +250,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int stage, bool doGPU, int lane); std::pair TPCClusterizerDecodeZSCount(unsigned int iSlice, unsigned int minTime, unsigned int maxTime); void RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, char mergeMode, GPUReconstruction::krnlDeviceType deviceType); + void RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType); std::atomic_flag mLockAtomic = ATOMIC_FLAG_INIT; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index a32506707c27e..414b546894e2a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -229,6 +229,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mBorderMemory, 2 * mNMaxSliceTracks); computePointerWithAlignment(mem, mBorderRangeMemory, 2 * mNMaxSliceTracks); computePointerWithAlignment(mem, mTrackLinks, mNMaxSliceTracks); + computePointerWithAlignment(mem, mTrackCCRoots, mNMaxSliceTracks); size_t tmpSize = CAMath::Max(CAMath::Max(mNMaxSingleSliceTracks, 1) * NSLICES * sizeof(int), CAMath::nextMultipleOf<4>(mNMaxTracks) * sizeof(int) + mNMaxClusters * sizeof(unsigned int)); computePointerWithAlignment(mem, mTmpMem, (tmpSize + sizeof(*mTmpMem) - 1) / sizeof(*mTmpMem)); @@ -901,7 +902,73 @@ GPUd() void GPUTPCGMMerger::MergeSlicesPrepare(int nBlocks, int nThreads, int iB MakeBorderTracks((nBlocks + 1) >> 1, nThreads, iBlock >> 1, iThread, border, b, n, useOrigTrackParam); } -GPUd() void GPUTPCGMMerger::ResolveMergeSlices(int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll) +GPUdi() void GPUTPCGMMerger::setBlockRange(int elems, int nBlocks, int iBlock, int& start, int& end) +{ + start = (elems + nBlocks - 1) / nBlocks * iBlock; + end = (elems + nBlocks - 1) / nBlocks * (iBlock + 1); + end = CAMath::Min(elems, end); +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsSetup(int nBlocks, int nThreads, int iBlock, int iThread) +{ + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int i = start + iThread; i < end; i += nThreads) { + mTrackCCRoots[i] = i; + } +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsHook(int nBlocks, int nThreads, int iBlock, int iThread) +{ + // Compute connected components in parallel, step 1. Source: Adaptive Work-Efficient Connected Components onthe GPU, Sutton et al, 2016 (https://arxiv.org/pdf/1612.01178.pdf) + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int itr = start + iThread; itr < end; itr += nThreads) { + int u = itr; + int v = mTrackLinks[u]; + if (v < 0) { + continue; + } + while (true) { + u = mTrackCCRoots[u]; + v = mTrackCCRoots[v]; + if (u == v) { + break; + } + int h = CAMath::Max(u, v); + int l = CAMath::Min(u, v); + + int old = CAMath::AtomicCAS(&mTrackCCRoots[h], h, l); + if (old == h) { + break; + } + + u = mTrackCCRoots[h]; + v = l; + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveFindConnectedComponentsMultiJump(int nBlocks, int nThreads, int iBlock, int iThread) +{ + // Compute connected components in parallel, step 2. + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + for (int itr = start + iThread; itr < end; itr += nThreads) { + int root = itr; + int next = mTrackCCRoots[root]; + if (root == next) { + continue; + } + do { + root = next; + next = mTrackCCRoots[next]; + } while (root != next); + mTrackCCRoots[itr] = root; + } +} + +GPUd() void GPUTPCGMMerger::ResolveMergeSlices(GPUResolveSharedMemory& smem, int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll) { if (!mergeAll) { /*int neighborType = useOrigTrackParam ? 1 : 0; @@ -933,137 +1000,171 @@ GPUd() void GPUTPCGMMerger::ResolveMergeSlices(int nBlocks, int nThreads, int iB newTrack2.SetPrevNeighbour( itr, neighborType );*/ } - for (int itr = 0; itr < SliceTrackInfoLocalTotal(); itr++) { - int itr2 = mTrackLinks[itr]; - if (itr2 < 0) { - continue; + int start, end; + setBlockRange(SliceTrackInfoLocalTotal(), nBlocks, iBlock, start, end); + + for (int baseIdx = 0; baseIdx < SliceTrackInfoLocalTotal(); baseIdx += nThreads) { + int itr = baseIdx + iThread; + bool inRange = itr < SliceTrackInfoLocalTotal(); + + int itr2 = -1; + if (inRange) { + itr2 = mTrackLinks[itr]; + } + + bool resolveSlice = (itr2 > -1); + if (resolveSlice) { + int root = mTrackCCRoots[itr]; + resolveSlice &= (start <= root) && (root < end); } - GPUTPCGMSliceTrack* track1 = &mSliceTrackInfos[itr]; - GPUTPCGMSliceTrack* track2 = &mSliceTrackInfos[itr2]; - GPUTPCGMSliceTrack* track1Base = track1; - GPUTPCGMSliceTrack* track2Base = track2; - bool sameSegment = CAMath::Abs(track1->NClusters() > track2->NClusters() ? track1->QPt() : track2->QPt()) < 2 || track1->QPt() * track2->QPt() > 0; - // GPUInfo("\nMerge %d with %d - same segment %d", itr, itr2, (int) sameSegment); - // PrintMergeGraph(track1, std::cout); - // PrintMergeGraph(track2, std::cout); + short smemIdx = work_group_scan_inclusive_add(short(resolveSlice)); - while (track2->PrevSegmentNeighbour() >= 0) { - track2 = &mSliceTrackInfos[track2->PrevSegmentNeighbour()]; + if (resolveSlice) { + smem.iTrack1[smemIdx - 1] = itr; + smem.iTrack2[smemIdx - 1] = itr2; } - if (sameSegment) { - if (track1 == track2) { - continue; + GPUbarrier(); + + if (iThread < nThreads - 1) { + continue; + } + + const int nSlices = smemIdx; + + for (int i = 0; i < nSlices; i++) { + itr = smem.iTrack1[i]; + itr2 = smem.iTrack2[i]; + + GPUTPCGMSliceTrack* track1 = &mSliceTrackInfos[itr]; + GPUTPCGMSliceTrack* track2 = &mSliceTrackInfos[itr2]; + GPUTPCGMSliceTrack* track1Base = track1; + GPUTPCGMSliceTrack* track2Base = track2; + + bool sameSegment = CAMath::Abs(track1->NClusters() > track2->NClusters() ? track1->QPt() : track2->QPt()) < 2 || track1->QPt() * track2->QPt() > 0; + // GPUInfo("\nMerge %d with %d - same segment %d", itr, itr2, (int) sameSegment); + // PrintMergeGraph(track1, std::cout); + // PrintMergeGraph(track2, std::cout); + + while (track2->PrevSegmentNeighbour() >= 0) { + track2 = &mSliceTrackInfos[track2->PrevSegmentNeighbour()]; } - while (track1->PrevSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + if (sameSegment) { if (track1 == track2) { - goto NextTrack; + continue; } - } - GPUCommonAlgorithm::swap(track1, track1Base); - for (int k = 0; k < 2; k++) { - GPUTPCGMSliceTrack* tmp = track1Base; - while (tmp->Neighbour(k) >= 0) { - tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; - if (tmp == track2) { + while (track1->PrevSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + if (track1 == track2) { goto NextTrack; } } - } - - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; - if (track1 == track2) { - goto NextTrack; + GPUCommonAlgorithm::swap(track1, track1Base); + for (int k = 0; k < 2; k++) { + GPUTPCGMSliceTrack* tmp = track1Base; + while (tmp->Neighbour(k) >= 0) { + tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; + if (tmp == track2) { + goto NextTrack; + } + } } - } - } else { - while (track1->PrevSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; - } - if (track1 == track2) { - continue; - } - for (int k = 0; k < 2; k++) { - GPUTPCGMSliceTrack* tmp = track1; - while (tmp->Neighbour(k) >= 0) { - tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; - if (tmp == track2) { + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + if (track1 == track2) { goto NextTrack; } } - } + } else { + while (track1->PrevSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->PrevSegmentNeighbour()]; + } - float z1min, z1max, z2min, z2max; - z1min = track1->MinClusterZT(); - z1max = track1->MaxClusterZT(); - z2min = track2->MinClusterZT(); - z2max = track2->MaxClusterZT(); - if (track1 != track1Base) { - z1min = CAMath::Min(z1min, track1Base->MinClusterZT()); - z1max = CAMath::Max(z1max, track1Base->MaxClusterZT()); - } - if (track2 != track2Base) { - z2min = CAMath::Min(z2min, track2Base->MinClusterZT()); - z2max = CAMath::Max(z2max, track2Base->MaxClusterZT()); - } - bool goUp = z2max - z1min > z1max - z2min; + if (track1 == track2) { + continue; + } + for (int k = 0; k < 2; k++) { + GPUTPCGMSliceTrack* tmp = track1; + while (tmp->Neighbour(k) >= 0) { + tmp = &mSliceTrackInfos[tmp->Neighbour(k)]; + if (tmp == track2) { + goto NextTrack; + } + } + } - if (track1->Neighbour(goUp) < 0 && track2->Neighbour(!goUp) < 0) { - track1->SetNeighbor(track2 - mSliceTrackInfos, goUp); - track2->SetNeighbor(track1 - mSliceTrackInfos, !goUp); - // GPUInfo("Result (simple neighbor)"); - // PrintMergeGraph(track1, std::cout); - continue; - } else if (track1->Neighbour(goUp) < 0) { - track2 = &mSliceTrackInfos[track2->Neighbour(!goUp)]; - GPUCommonAlgorithm::swap(track1, track2); - } else if (track2->Neighbour(!goUp) < 0) { - track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; - } else { // Both would work, but we use the simpler one - track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + float z1min, z1max, z2min, z2max; + z1min = track1->MinClusterZT(); + z1max = track1->MaxClusterZT(); + z2min = track2->MinClusterZT(); + z2max = track2->MaxClusterZT(); + if (track1 != track1Base) { + z1min = CAMath::Min(z1min, track1Base->MinClusterZT()); + z1max = CAMath::Max(z1max, track1Base->MaxClusterZT()); + } + if (track2 != track2Base) { + z2min = CAMath::Min(z2min, track2Base->MinClusterZT()); + z2max = CAMath::Max(z2max, track2Base->MaxClusterZT()); + } + bool goUp = z2max - z1min > z1max - z2min; + + if (track1->Neighbour(goUp) < 0 && track2->Neighbour(!goUp) < 0) { + track1->SetNeighbor(track2 - mSliceTrackInfos, goUp); + track2->SetNeighbor(track1 - mSliceTrackInfos, !goUp); + // GPUInfo("Result (simple neighbor)"); + // PrintMergeGraph(track1, std::cout); + continue; + } else if (track1->Neighbour(goUp) < 0) { + track2 = &mSliceTrackInfos[track2->Neighbour(!goUp)]; + GPUCommonAlgorithm::swap(track1, track2); + } else if (track2->Neighbour(!goUp) < 0) { + track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + } else { // Both would work, but we use the simpler one + track1 = &mSliceTrackInfos[track1->Neighbour(goUp)]; + } + track1Base = track1; } - track1Base = track1; - } - track2Base = track2; - if (!sameSegment) { - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + track2Base = track2; + if (!sameSegment) { + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + } } - } - track1->SetNextSegmentNeighbour(track2 - mSliceTrackInfos); - track2->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); - for (int k = 0; k < 2; k++) { - track1 = track1Base; - track2 = track2Base; - while (track2->Neighbour(k) >= 0) { - if (track1->Neighbour(k) >= 0) { - GPUTPCGMSliceTrack* track1new = &mSliceTrackInfos[track1->Neighbour(k)]; - GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; - track2->SetNeighbor(-1, k); - track2new->SetNeighbor(-1, k ^ 1); - track1 = track1new; - while (track1->NextSegmentNeighbour() >= 0) { - track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + track1->SetNextSegmentNeighbour(track2 - mSliceTrackInfos); + track2->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); + // k = 0: Merge right side + // k = 1: Merge left side + for (int k = 0; k < 2; k++) { + track1 = track1Base; + track2 = track2Base; + while (track2->Neighbour(k) >= 0) { + if (track1->Neighbour(k) >= 0) { + GPUTPCGMSliceTrack* track1new = &mSliceTrackInfos[track1->Neighbour(k)]; + GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; + track2->SetNeighbor(-1, k); + track2new->SetNeighbor(-1, k ^ 1); + track1 = track1new; + while (track1->NextSegmentNeighbour() >= 0) { + track1 = &mSliceTrackInfos[track1->NextSegmentNeighbour()]; + } + track1->SetNextSegmentNeighbour(track2new - mSliceTrackInfos); + track2new->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); + track1 = track1new; + track2 = track2new; + } else { + GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; + track1->SetNeighbor(track2->Neighbour(k), k); + track2->SetNeighbor(-1, k); + track2new->SetNeighbor(track1 - mSliceTrackInfos, k ^ 1); } - track1->SetNextSegmentNeighbour(track2new - mSliceTrackInfos); - track2new->SetPrevSegmentNeighbour(track1 - mSliceTrackInfos); - track1 = track1new; - track2 = track2new; - } else { - GPUTPCGMSliceTrack* track2new = &mSliceTrackInfos[track2->Neighbour(k)]; - track1->SetNeighbor(track2->Neighbour(k), k); - track2->SetNeighbor(-1, k); - track2new->SetNeighbor(track1 - mSliceTrackInfos, k ^ 1); } } + // GPUInfo("Result"); + // PrintMergeGraph(track1, std::cout); + NextTrack:; } - // GPUInfo("Result"); - // PrintMergeGraph(track1, std::cout); - NextTrack:; } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index ad72cfc65532e..2185cb10654f9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -22,6 +22,7 @@ #include "GPUCommonDef.h" #include "GPUProcessor.h" #include "GPUTPCGMMergerTypes.h" +#include "GPUGeneralKernels.h" #if !defined(GPUCA_GPUCODE) #include @@ -79,6 +80,11 @@ class GPUTPCGMMerger : public GPUProcessor unsigned char leg; }; + struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { + int iTrack1[GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCGMMergerResolve_step3)]; + int iTrack2[GPUCA_GET_THREAD_COUNT(GPUCA_LB_GPUTPCGMMergerResolve_step3)]; + }; + void InitializeProcessor(); void RegisterMemoryAllocation(); void SetMaxData(const GPUTrackingInOutPointers& io); @@ -149,7 +155,10 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void Finalize0(int nBlocks, int nThreads, int iBlock, int iThread); GPUd() void Finalize1(int nBlocks, int nThreads, int iBlock, int iThread); GPUd() void Finalize2(int nBlocks, int nThreads, int iBlock, int iThread); - GPUd() void ResolveMergeSlices(int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll); + GPUd() void ResolveFindConnectedComponentsSetup(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveFindConnectedComponentsHook(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveFindConnectedComponentsMultiJump(int nBlocks, int nThreads, int iBlock, int iThread); + GPUd() void ResolveMergeSlices(GPUResolveSharedMemory& smem, int nBlocks, int nThreads, int iBlock, int iThread, char useOrigTrackParam, char mergeAll); #ifndef GPUCA_GPUCODE void DumpSliceTracks(std::ostream& out); @@ -185,12 +194,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUdi() int SliceTrackInfoLocalTotal() { return mSliceTrackInfoIndex[NSLICES]; } GPUdi() int SliceTrackInfoTotal() { return mSliceTrackInfoIndex[2 * NSLICES]; } + GPUdi() void setBlockRange(int elems, int nBlocks, int iBlock, int& start, int& end); + int mNextSliceInd[NSLICES]; int mPrevSliceInd[NSLICES]; const GPUTPCSliceOutput* mkSlices[NSLICES]; //* array of input slice tracks int* mTrackLinks; + int* mTrackCCRoots; // root of the connected component of this track unsigned int mNMaxSliceTracks; // maximum number of incoming slice tracks unsigned int mNMaxTracks; // maximum number of output tracks @@ -201,7 +213,7 @@ class GPUTPCGMMerger : public GPUProcessor unsigned short mMemoryResMemory; unsigned short mMemoryResOutput; - int mNClusters; // Total number of incoming clusters (from slice tracks) + int mNClusters; // Total number of incoming clusters (from slice tracks) GPUTPCGMMergedTrack* mOutputTracks; //* array of output merged tracks GPUTPCGMSliceTrack* mSliceTrackInfos; //* additional information for slice tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 508a169cca26d..ec8a4456b7adf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -12,6 +12,7 @@ /// \author David Rohr #include "GPUTPCGMMergerGPU.h" +#include "GPUCommonAlgorithm.h" #if defined(WITH_OPENMP) && !defined(GPUCA_GPUCODE) #include "GPUReconstruction.h" #endif @@ -66,12 +67,27 @@ GPUdii() void GPUTPCGMMergerUnpackSaveNumber::Thread<0>(int nBlocks, int nThread } template <> -GPUdii() void GPUTPCGMMergerResolve::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger, char useOrigTrackParam, char mergeAll) +GPUdii() void GPUTPCGMMergerResolve::Thread<0>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) { - if (iThread || iBlock) { - return; - } - merger.ResolveMergeSlices(nBlocks, nThreads, iBlock, iThread, useOrigTrackParam, mergeAll); + merger.ResolveFindConnectedComponentsSetup(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<1>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) +{ + merger.ResolveFindConnectedComponentsHook(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<2>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger) +{ + merger.ResolveFindConnectedComponentsMultiJump(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerResolve::Thread<3>(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& GPUrestrict() smem, processorType& GPUrestrict() merger, char useOrigTrackParam, char mergeAll) +{ + merger.ResolveMergeSlices(smem, nBlocks, nThreads, iBlock, iThread, useOrigTrackParam, mergeAll); } template <> diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 3906e66f0e62e..bb2cde4f68060 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -91,9 +91,12 @@ class GPUTPCGMMergerUnpackResetIds : public GPUTPCGMMergerGeneral class GPUTPCGMMergerResolve : public GPUTPCGMMergerGeneral { public: -#if !defined(GPUCA_ALIROOT_LIB) || !defined(GPUCA_GPUCODE) - template - GPUd() static void Thread(int nBlocks, int nThreads, int iBlock, int iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, char useOrigTrackParam, char mergeAll); + struct GPUSharedMemory : public GPUTPCGMMerger::GPUResolveSharedMemory { + }; + +#if !defined(GPUCA_ALIROOT_LIB) + template + GPUd() static void Thread(int nBlocks, int nThreads, int iBlock, int iThread, GPUSharedMemory& smem, processorType& clusterer, Args... args); #endif };