GPU: Parallize track merger resolve kernel. (#3676)

Author: fweig

GPU/Common/GPUDefGPUParameters.h

@@ -45,7 +45,10 @@
   #define GPUCA_LB_GPUTPCGMMergerSliceRefit 256
   #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256
   #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
-  #define GPUCA_LB_GPUTPCGMMergerResolve 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
   #define GPUCA_LB_GPUTPCGMMergerClearLinks 256
   #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256
   #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256
@@ -90,7 +93,10 @@
   #define GPUCA_LB_GPUTPCGMMergerSliceRefit 64
   #define GPUCA_LB_GPUTPCGMMergerUnpackResetIds 256
   #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
-  #define GPUCA_LB_GPUTPCGMMergerResolve 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
+  #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
   #define GPUCA_LB_GPUTPCGMMergerClearLinks 256
   #define GPUCA_LB_GPUTPCGMMergerMergeWithinPrepare 256
   #define GPUCA_LB_GPUTPCGMMergerMergeSlicesPrepare 256, 2
@@ -186,8 +192,17 @@
   #ifndef GPUCA_LB_GPUTPCGMMergerUnpackGlobal
     #define GPUCA_LB_GPUTPCGMMergerUnpackGlobal 256
   #endif
-  #ifndef GPUCA_LB_GPUTPCGMMergerResolve
-    #define GPUCA_LB_GPUTPCGMMergerResolve 256
+  #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step0
+    #define GPUCA_LB_GPUTPCGMMergerResolve_step0 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step1
+    #define GPUCA_LB_GPUTPCGMMergerResolve_step1 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step2
+    #define GPUCA_LB_GPUTPCGMMergerResolve_step2 256
+  #endif
+  #ifndef GPUCA_LB_GPUTPCGMMergerResolve_step3
+    #define GPUCA_LB_GPUTPCGMMergerResolve_step3 256
   #endif
   #ifndef GPUCA_LB_GPUTPCGMMergerClearLinks
     #define GPUCA_LB_GPUTPCGMMergerClearLinks 256

GPU/GPUTracking/Base/GPUReconstructionKernels.h

@@ -33,7 +33,10 @@ GPUCA_KRNL_LB((GPUTPCGMMergerUnpackResetIds                 ), (simple), (, int
 GPUCA_KRNL_LB((GPUTPCGMMergerSliceRefit                     ), (simple), (, int iSlice), (, iSlice))
 GPUCA_KRNL_LB((GPUTPCGMMergerUnpackGlobal                   ), (simple), (, int iSlice), (, iSlice))
 GPUCA_KRNL((   GPUTPCGMMergerUnpackSaveNumber               ), (simple), (, int id), (, id))
-GPUCA_KRNL_LB((GPUTPCGMMergerResolve                        ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll))
+GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step0                 ), (simple), (), ())
+GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step1                 ), (simple), (), ())
+GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step2                 ), (simple), (), ())
+GPUCA_KRNL_LB((GPUTPCGMMergerResolve, step3                 ), (simple), (, char useOrigTrackParam, char mergeAll), (, useOrigTrackParam, mergeAll))
 GPUCA_KRNL_LB((GPUTPCGMMergerClearLinks                     ), (simple), (, char nOutput), (, nOutput))
 GPUCA_KRNL_LB((GPUTPCGMMergerMergeWithinPrepare             ), (simple), (), ())
 GPUCA_KRNL_LB((GPUTPCGMMergerMergeSlicesPrepare             ), (simple), (, int border0, int border1, char useOrigTrackParam), (, border0, border1, useOrigTrackParam))

GPU/GPUTracking/Global/GPUChainTracking.cxx

@@ -1643,6 +1643,15 @@ void GPUChainTracking::RunTPCTrackingMerger_MergeBorderTracks(char withinSlice,
   mRec->ReturnVolatileDeviceMemory();
 }
 
+void GPUChainTracking::RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType)
+{
+  runKernel<GPUTPCGMMergerResolve, 0>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
+  runKernel<GPUTPCGMMergerResolve, 1>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
+  runKernel<GPUTPCGMMergerResolve, 2>(GetGridBlk(BlockCount(), 0, deviceType), krnlRunRangeNone, krnlEventNone);
+  // TODO: Determine number of blocks from block size to fully occupy gpu
+  runKernel<GPUTPCGMMergerResolve, 3>(GetGridBlk(BlockCount() * 4, 0, deviceType), krnlRunRangeNone, krnlEventNone, useOrigTrackParam, mergeAll);
+}
+
 int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
 {
   if (GetDeviceProcessingSettings().debugLevel >= 6 && GetDeviceProcessingSettings().comparableDebutOutput && param().rec.mergerReadFromTrackerDirectly) {
@@ -1701,22 +1710,22 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
   runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
   runKernel<GPUTPCGMMergerMergeWithinPrepare>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone);
   RunTPCTrackingMerger_MergeBorderTracks(1, 0, deviceType);
-  runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1);
+  RunTPCTrackingMerger_Resolve(0, 1, deviceType);
   DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedWithinSlices, mDebugFile);
 
   runKernel<GPUTPCGMMergerClearLinks>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0);
   runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
   runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 2, 3, 0);
   RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType);
-  runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0);
+  RunTPCTrackingMerger_Resolve(0, 1, deviceType);
   runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
   runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 0);
   RunTPCTrackingMerger_MergeBorderTracks(0, 0, deviceType);
-  runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 0);
+  RunTPCTrackingMerger_Resolve(0, 1, deviceType);
   runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));
   runKernel<GPUTPCGMMergerMergeSlicesPrepare>(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), krnlRunRangeNone, krnlEventNone, 0, 1, 1);
   RunTPCTrackingMerger_MergeBorderTracks(0, -1, deviceType);
-  runKernel<GPUTPCGMMergerResolve>(GetGridBlk(numBlocks, 0, deviceType), krnlRunRangeNone, krnlEventNone, 1, 0);
+  RunTPCTrackingMerger_Resolve(0, 1, deviceType);
   DoDebugAndDump(RecoStep::TPCMerging, 0, doGPUall, Merger, &GPUTPCGMMerger::DumpMergedBetweenSlices, mDebugFile);
 
   runKernel<GPUMemClean16>({1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}, krnlRunRangeNone, {}, MergerShadowAll.TmpCounter(), 2 * NSLICES * sizeof(*MergerShadowAll.TmpCounter()));

GPU/GPUTracking/Global/GPUChainTracking.h

@@ -250,6 +250,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega
   void RunTPCClusterizer_compactPeaks(GPUTPCClusterFinder& clusterer, GPUTPCClusterFinder& clustererShadow, int stage, bool doGPU, int lane);
   std::pair<unsigned int, unsigned int> TPCClusterizerDecodeZSCount(unsigned int iSlice, unsigned int minTime, unsigned int maxTime);
   void RunTPCTrackingMerger_MergeBorderTracks(char withinSlice, char mergeMode, GPUReconstruction::krnlDeviceType deviceType);
+  void RunTPCTrackingMerger_Resolve(char useOrigTrackParam, char mergeAll, GPUReconstruction::krnlDeviceType deviceType);
 
   std::atomic_flag mLockAtomic = ATOMIC_FLAG_INIT;